2024-08-14T06:42:00.185 INFO:root:teuthology version: 0.0.1.dev425+g43b8805 2024-08-14T06:42:00.207 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-14T06:42:00.302 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/pdonnell-2024-08-14_02:13:41-fs-wip-pdonnell-testing-20240813.125846-debug-distro-default-smithi/7854349 branch: wip-pdonnell-testing-20240813.125846-debug description: fs/functional/{begin/{0-install 1-ceph 2-logrotate 3-modules} clusters/1a3s-mds-4c-client conf/{client mds mgr mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{centos_9.stream k-stock} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile pg_health} subvol_versions/create_subvol_version_v2 tasks/alternate-pool} email: pdonnell@redhat.com first_in_suite: false job_id: '7854349' kernel: client: sha1: distro kdb: 1 sha1: distro ktype: distro last_in_suite: false log-rotate: ceph-mds: 10G ceph-osd: 10G machine_type: smithi mgrmodules: sequential: - print: Enabling mgr modules name: pdonnell-2024-08-14_02:13:41-fs-wip-pdonnell-testing-20240813.125846-debug-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 20 - machine: disk: 200 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-pdonnell-testing-20240813.125846-debug ceph: cluster-conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 mds: debug mds: 20 debug mds balancer: 20 debug ms: 2 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mon: mon op complaint time: 120 mon warn on pool no app: false osd: osd op complaint time: 180 conf: client: pid file: '' global: ms die on skipped message: false mgr: debug client: 20 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon down mkfs grace: 300 osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - fs.*is degraded - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - insufficient standby MDS daemons available - MDS_UP_LESS_THAN_MAX - online, but wants - filesystem is online with fewer MDS than max_mds - POOL_APP_NOT_ENABLED - do not have an application enabled - overall HEALTH_ - Replacing daemon - deprecated feature inline_data - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding - PG_AVAILABILITY - PG_DEGRADED - Reduced data availability - Degraded data redundancy - bad backtrace - object missing on disk - error reading table object - error reading sessionmap - unmatched fragstat - unmatched rstat - was unreadable, recreating it now - Scrub error on inode - Metadata damage detected - MDS_FAILED - MDS_DAMAGE sha1: 488c4d2c8097ced3f3bab85cfc6687562b3affe3 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 488c4d2c8097ced3f3bab85cfc6687562b3affe3 kclient: syntax: v2 subvolume_version: 2 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-pdonnell-testing-20240813.125846-debug sha1: 488c4d2c8097ced3f3bab85cfc6687562b3affe3 owner: scheduled_pdonnell@teuthology priority: 90 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mgr.y - mds.a - mds.c - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - client.2 - - mon.b - mon.c - mgr.x - mds.b - mds.d - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - client.3 seed: 8073 sha1: 488c4d2c8097ced3f3bab85cfc6687562b3affe3 sleep_before_teardown: 0 subset: 511/512 suite: fs suite_branch: wip-pdonnell-testing-20240813.125846-debug suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_488c4d2c8097ced3f3bab85cfc6687562b3affe3/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 488c4d2c8097ced3f3bab85cfc6687562b3affe3 syslog: ignorelist: - WARNING*.*check_session_state - WARNING*.*__ceph_remove_cap targets: smithi098.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLEZbyx2EaBgUyzjuo7qSCWQMbTf+yHkY6kOaVcpALjZki/tddSd1y1TEuAJDn4q0YIK+oIzlGSC0ekfTswZKvo= smithi191.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHAEnArlr9jbW6v7LWN48BdymRadcg88rHYqyEmE5nS8+HvlARffhhz9joA2m0wa7kXPJrl9aRLdPl7FgU/Mrko= tasks: - install: extra_packages: deb: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror rpm: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror extra_system_packages: deb: - bison - flex - libelf-dev - libssl-dev - network-manager - iproute2 - util-linux - dump - indent - libaio-dev - libtool-bin - uuid-dev - xfslibs-dev - postgresql - postgresql-client - postgresql-common - postgresql-contrib rpm: - bison - flex - elfutils-libelf-devel - openssl-devel - NetworkManager - iproute - util-linux - libacl-devel - libaio-devel - libattr-devel - libtool - libuuid-devel - xfsdump - xfsprogs - xfsprogs-devel - libaio-devel - libtool - libuuid-devel - xfsprogs-devel - postgresql - postgresql-server - postgresql-contrib - ceph: create_rbd_pool: false - sequential: - mgrmodules - kclient: null - cephfs_test_runner: modules: - tasks.cephfs.test_recovery_pool teuthology: fragments_dropped: [] meta: {} postmerge: - local function is_kupstream() return yaml.ktype == 'upstream' end - local function is_kdistro() return yaml.ktype == 'distro' end - local function is_fuse() return false end - local function syntax_version() return yaml.overrides.kclient.syntax end teuthology_branch: main teuthology_sha1: 43b88056bcf6a0548bdd2d0502e56290cbae52df timestamp: 2024-08-14_02:13:41 tube: smithi user: pdonnell verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-08-14T06:42:00.302 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_488c4d2c8097ced3f3bab85cfc6687562b3affe3/qa; will attempt to use it 2024-08-14T06:42:00.303 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_488c4d2c8097ced3f3bab85cfc6687562b3affe3/qa/tasks 2024-08-14T06:42:00.303 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-08-14T06:42:00.305 INFO:teuthology.task.internal:Checking packages... 2024-08-14T06:42:00.326 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '488c4d2c8097ced3f3bab85cfc6687562b3affe3' 2024-08-14T06:42:00.326 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-14T06:42:00.326 INFO:teuthology.packaging:ref: None 2024-08-14T06:42:00.327 INFO:teuthology.packaging:tag: None 2024-08-14T06:42:00.327 INFO:teuthology.packaging:branch: wip-pdonnell-testing-20240813.125846-debug 2024-08-14T06:42:00.327 INFO:teuthology.packaging:sha1: 488c4d2c8097ced3f3bab85cfc6687562b3affe3 2024-08-14T06:42:00.327 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-pdonnell-testing-20240813.125846-debug 2024-08-14T06:42:00.505 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-5779.g488c4d2c 2024-08-14T06:42:00.507 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-08-14T06:42:00.513 INFO:teuthology.task.internal:no buildpackages task found 2024-08-14T06:42:00.513 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-08-14T06:42:00.585 INFO:teuthology.task.internal:Saving configuration 2024-08-14T06:42:00.602 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-08-14T06:42:00.618 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-08-14T06:42:00.652 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi098.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/pdonnell-2024-08-14_02:13:41-fs-wip-pdonnell-testing-20240813.125846-debug-distro-default-smithi/7854349', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-14 06:36:54.670052', 'locked_by': 'scheduled_pdonnell@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLEZbyx2EaBgUyzjuo7qSCWQMbTf+yHkY6kOaVcpALjZki/tddSd1y1TEuAJDn4q0YIK+oIzlGSC0ekfTswZKvo='} 2024-08-14T06:42:00.711 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi191.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/pdonnell-2024-08-14_02:13:41-fs-wip-pdonnell-testing-20240813.125846-debug-distro-default-smithi/7854349', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-14 06:36:54.671475', 'locked_by': 'scheduled_pdonnell@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHAEnArlr9jbW6v7LWN48BdymRadcg88rHYqyEmE5nS8+HvlARffhhz9joA2m0wa7kXPJrl9aRLdPl7FgU/Mrko='} 2024-08-14T06:42:00.711 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-08-14T06:42:00.718 INFO:teuthology.task.internal:roles: ubuntu@smithi098.front.sepia.ceph.com - ['mon.a', 'mgr.y', 'mds.a', 'mds.c', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'client.2'] 2024-08-14T06:42:00.718 INFO:teuthology.task.internal:roles: ubuntu@smithi191.front.sepia.ceph.com - ['mon.b', 'mon.c', 'mgr.x', 'mds.b', 'mds.d', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'client.3'] 2024-08-14T06:42:00.718 INFO:teuthology.run_tasks:Running task console_log... 2024-08-14T06:42:00.821 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f5cb5d77ac0>, signals=[15]) 2024-08-14T06:42:00.822 INFO:teuthology.run_tasks:Running task internal.connect... 2024-08-14T06:42:00.828 INFO:teuthology.task.internal:Opening connections... 2024-08-14T06:42:00.828 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi098.front.sepia.ceph.com 2024-08-14T06:42:00.829 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-14T06:42:00.912 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi191.front.sepia.ceph.com 2024-08-14T06:42:00.913 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-14T06:42:00.998 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-08-14T06:42:01.005 DEBUG:teuthology.orchestra.run.smithi098:> uname -m 2024-08-14T06:42:01.023 INFO:teuthology.orchestra.run.smithi098.stdout:x86_64 2024-08-14T06:42:01.023 DEBUG:teuthology.orchestra.run.smithi098:> cat /etc/os-release 2024-08-14T06:42:01.080 INFO:teuthology.orchestra.run.smithi098.stdout:NAME="CentOS Stream" 2024-08-14T06:42:01.080 INFO:teuthology.orchestra.run.smithi098.stdout:VERSION="9" 2024-08-14T06:42:01.080 INFO:teuthology.orchestra.run.smithi098.stdout:ID="centos" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:ID_LIKE="rhel fedora" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:VERSION_ID="9" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:PLATFORM_ID="platform:el9" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:ANSI_COLOR="0;31" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:LOGO="fedora-logo-icon" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:HOME_URL="https://centos.org/" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-14T06:42:01.081 INFO:teuthology.orchestra.run.smithi098.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-14T06:42:01.082 INFO:teuthology.lock.ops:Updating smithi098.front.sepia.ceph.com on lock server 2024-08-14T06:42:01.115 DEBUG:teuthology.orchestra.run.smithi191:> uname -m 2024-08-14T06:42:01.134 INFO:teuthology.orchestra.run.smithi191.stdout:x86_64 2024-08-14T06:42:01.134 DEBUG:teuthology.orchestra.run.smithi191:> cat /etc/os-release 2024-08-14T06:42:01.190 INFO:teuthology.orchestra.run.smithi191.stdout:NAME="CentOS Stream" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION="9" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:ID="centos" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:ID_LIKE="rhel fedora" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION_ID="9" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:PLATFORM_ID="platform:el9" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:ANSI_COLOR="0;31" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:LOGO="fedora-logo-icon" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:HOME_URL="https://centos.org/" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-14T06:42:01.191 INFO:teuthology.orchestra.run.smithi191.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-14T06:42:01.192 INFO:teuthology.lock.ops:Updating smithi191.front.sepia.ceph.com on lock server 2024-08-14T06:42:01.224 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-08-14T06:42:01.234 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-08-14T06:42:01.238 INFO:teuthology.task.internal:Checking for old test directory... 2024-08-14T06:42:01.238 DEBUG:teuthology.orchestra.run.smithi098:> test '!' -e /home/ubuntu/cephtest 2024-08-14T06:42:01.241 DEBUG:teuthology.orchestra.run.smithi191:> test '!' -e /home/ubuntu/cephtest 2024-08-14T06:42:01.256 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-08-14T06:42:01.263 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-08-14T06:42:01.263 DEBUG:teuthology.orchestra.run.smithi098:> test -z $(ls -A /var/lib/ceph) 2024-08-14T06:42:01.298 DEBUG:teuthology.orchestra.run.smithi191:> test -z $(ls -A /var/lib/ceph) 2024-08-14T06:42:01.398 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-08-14T06:42:01.459 INFO:teuthology.run_tasks:Running task kernel... 2024-08-14T06:42:01.475 INFO:teuthology.task.kernel:normalize config orig: {'client': {'sha1': 'distro'}, 'kdb': 1, 'sha1': 'distro'} 2024-08-14T06:42:01.475 INFO:teuthology.task.kernel:normalize config final: {'client.0': {'sha1': 'distro'}, 'client.2': {'sha1': 'distro'}, 'client.1': {'sha1': 'distro'}, 'client.3': {'sha1': 'distro'}} 2024-08-14T06:42:01.475 INFO:teuthology.task.kernel:config {'client.0': {'sha1': 'distro'}, 'client.1': {'sha1': 'distro'}}, timeout 300 2024-08-14T06:42:01.476 DEBUG:teuthology.orchestra.run.smithi098:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-14T06:42:01.476 DEBUG:teuthology.orchestra.run.smithi191:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-14T06:42:01.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-14T06:42:01.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-14T06:42:01.493 DEBUG:teuthology.orchestra.run.smithi191:> uname -r 2024-08-14T06:42:01.494 DEBUG:teuthology.orchestra.run.smithi098:> uname -r 2024-08-14T06:42:01.550 INFO:teuthology.orchestra.run.smithi098.stdout:5.14.0-437.el9.x86_64 2024-08-14T06:42:01.551 INFO:teuthology.task.kernel:Running kernel on smithi098: 5.14.0-437.el9.x86_64 2024-08-14T06:42:01.551 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum install -y kernel 2024-08-14T06:42:01.552 INFO:teuthology.orchestra.run.smithi191.stdout:5.14.0-437.el9.x86_64 2024-08-14T06:42:01.552 INFO:teuthology.task.kernel:Running kernel on smithi191: 5.14.0-437.el9.x86_64 2024-08-14T06:42:01.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum install -y kernel 2024-08-14T06:42:05.315 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 9 - BaseOS 3.9 MB/s | 8.2 MB 00:02 2024-08-14T06:42:05.818 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - BaseOS 3.2 MB/s | 8.2 MB 00:02 2024-08-14T06:42:07.008 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 9 - AppStream 35 MB/s | 20 MB 00:00 2024-08-14T06:42:07.861 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - AppStream 21 MB/s | 20 MB 00:00 2024-08-14T06:42:11.125 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 9 - CRB 17 MB/s | 6.6 MB 00:00 2024-08-14T06:42:12.188 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.6 MB 00:00 2024-08-14T06:42:13.109 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 9 - Extras packages 31 kB/s | 18 kB 00:00 2024-08-14T06:42:13.837 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - Extras packages 63 kB/s | 18 kB 00:00 2024-08-14T06:42:13.980 INFO:teuthology.orchestra.run.smithi098.stdout:Extra Packages for Enterprise Linux 30 MB/s | 22 MB 00:00 2024-08-14T06:42:15.049 INFO:teuthology.orchestra.run.smithi191.stdout:Extra Packages for Enterprise Linux 21 MB/s | 22 MB 00:01 2024-08-14T06:42:19.353 INFO:teuthology.orchestra.run.smithi098.stdout:lab-extras 24 kB/s | 1.7 kB 00:00 2024-08-14T06:42:20.339 INFO:teuthology.orchestra.run.smithi191.stdout:lab-extras 27 kB/s | 1.7 kB 00:00 2024-08-14T06:42:21.451 INFO:teuthology.orchestra.run.smithi098.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-14T06:42:21.451 INFO:teuthology.orchestra.run.smithi098.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-08-14T06:42:21.547 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2024-08-14T06:42:21.558 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout: Package Architecture Version Repository Size 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout: kernel x86_64 5.14.0-480.el9 baseos 1.5 M 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-core x86_64 5.14.0-480.el9 baseos 17 M 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-modules x86_64 5.14.0-480.el9 baseos 36 M 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-modules-core x86_64 5.14.0-480.el9 baseos 30 M 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout:Install 4 Packages 2024-08-14T06:42:21.559 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:42:21.560 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 84 M 2024-08-14T06:42:21.560 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 126 M 2024-08-14T06:42:21.560 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2024-08-14T06:42:22.320 INFO:teuthology.orchestra.run.smithi191.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-14T06:42:22.320 INFO:teuthology.orchestra.run.smithi191.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-08-14T06:42:22.416 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-08-14T06:42:22.425 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-08-14T06:42:22.425 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2024-08-14T06:42:22.425 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-08-14T06:42:22.425 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-08-14T06:42:22.425 INFO:teuthology.orchestra.run.smithi191.stdout: kernel x86_64 5.14.0-480.el9 baseos 1.5 M 2024-08-14T06:42:22.425 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-08-14T06:42:22.426 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-core x86_64 5.14.0-480.el9 baseos 17 M 2024-08-14T06:42:22.426 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-modules x86_64 5.14.0-480.el9 baseos 36 M 2024-08-14T06:42:22.426 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-modules-core x86_64 5.14.0-480.el9 baseos 30 M 2024-08-14T06:42:22.426 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:42:22.426 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-08-14T06:42:22.426 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-08-14T06:42:22.426 INFO:teuthology.orchestra.run.smithi191.stdout:Install 4 Packages 2024-08-14T06:42:22.426 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:42:22.427 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 84 M 2024-08-14T06:42:22.427 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 126 M 2024-08-14T06:42:22.427 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-08-14T06:42:22.870 INFO:teuthology.orchestra.run.smithi098.stdout:(1/4): kernel-5.14.0-480.el9.x86_64.rpm 1.3 MB/s | 1.5 MB 00:01 2024-08-14T06:42:23.898 INFO:teuthology.orchestra.run.smithi191.stdout:(1/4): kernel-5.14.0-480.el9.x86_64.rpm 1.3 MB/s | 1.5 MB 00:01 2024-08-14T06:42:24.654 INFO:teuthology.orchestra.run.smithi098.stdout:(2/4): kernel-modules-5.14.0-480.el9.x86_64.rpm 12 MB/s | 36 MB 00:02 2024-08-14T06:42:24.854 INFO:teuthology.orchestra.run.smithi098.stdout:(3/4): kernel-core-5.14.0-480.el9.x86_64.rpm 5.4 MB/s | 17 MB 00:03 2024-08-14T06:42:25.590 INFO:teuthology.orchestra.run.smithi191.stdout:(2/4): kernel-core-5.14.0-480.el9.x86_64.rpm 6.0 MB/s | 17 MB 00:02 2024-08-14T06:42:25.915 INFO:teuthology.orchestra.run.smithi191.stdout:(3/4): kernel-modules-5.14.0-480.el9.x86_64.rpm 11 MB/s | 36 MB 00:03 2024-08-14T06:42:26.004 INFO:teuthology.orchestra.run.smithi098.stdout:(4/4): kernel-modules-core-5.14.0-480.el9.x86_6 9.6 MB/s | 30 MB 00:03 2024-08-14T06:42:26.004 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2024-08-14T06:42:26.005 INFO:teuthology.orchestra.run.smithi098.stdout:Total 19 MB/s | 84 MB 00:04 2024-08-14T06:42:26.432 INFO:teuthology.orchestra.run.smithi191.stdout:(4/4): kernel-modules-core-5.14.0-480.el9.x86_6 12 MB/s | 30 MB 00:02 2024-08-14T06:42:26.432 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-08-14T06:42:26.433 INFO:teuthology.orchestra.run.smithi191.stdout:Total 21 MB/s | 84 MB 00:04 2024-08-14T06:42:26.745 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2024-08-14T06:42:27.096 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2024-08-14T06:42:27.097 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2024-08-14T06:42:27.172 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-08-14T06:42:27.519 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-08-14T06:42:27.519 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-08-14T06:42:28.119 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2024-08-14T06:42:28.120 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2024-08-14T06:42:28.521 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-08-14T06:42:28.522 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-08-14T06:42:30.563 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2024-08-14T06:42:30.967 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-08-14T06:42:31.231 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : kernel-modules-core-5.14.0-480.el9.x86_64 1/4 2024-08-14T06:42:31.330 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-14T06:42:31.717 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-modules-core-5.14.0-480.el9.x86_64 1/4 2024-08-14T06:42:31.817 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-14T06:42:33.989 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-14T06:42:34.145 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-14T06:42:34.494 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-14T06:42:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-14T06:42:38.976 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-14T06:42:39.046 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : kernel-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:42:39.567 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-14T06:42:39.692 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:42:43.883 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:42:44.537 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:43:13.844 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:43:13.885 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:43:14.257 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:43:14.257 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : kernel-5.14.0-480.el9.x86_64 1/4 2024-08-14T06:43:14.258 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-14T06:43:14.258 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-14T06:43:14.480 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:43:14.519 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:43:14.912 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:43:14.913 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-5.14.0-480.el9.x86_64 1/4 2024-08-14T06:43:14.913 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-14T06:43:14.913 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-14T06:43:15.059 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:43:15.059 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:43:15.059 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2024-08-14T06:43:15.060 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-5.14.0-480.el9.x86_64 2024-08-14T06:43:15.060 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-core-5.14.0-480.el9.x86_64 2024-08-14T06:43:15.060 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-modules-5.14.0-480.el9.x86_64 2024-08-14T06:43:15.060 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-modules-core-5.14.0-480.el9.x86_64 2024-08-14T06:43:15.060 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:43:15.060 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2024-08-14T06:43:15.456 DEBUG:teuthology.orchestra.run.smithi098:> rpm -q kernel | sort -rV | head -n 1 2024-08-14T06:43:15.560 INFO:teuthology.orchestra.run.smithi098.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-14T06:43:15.561 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-14T06:43:15.561 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-480.el9.x86_64 2024-08-14T06:43:15.561 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-08-14T06:43:15.561 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-08-14T06:43:15.561 INFO:teuthology.task.kernel:Installing distro kernel on client.0... 2024-08-14T06:43:15.561 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi098.front.sepia.ceph.com, path=None, version=distro) 2024-08-14T06:43:15.561 DEBUG:teuthology.orchestra.run.smithi098:> rpm -q kernel | sort -rV | head -n 1 2024-08-14T06:43:15.618 INFO:teuthology.orchestra.run.smithi098.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-14T06:43:15.618 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-14T06:43:15.619 DEBUG:teuthology.orchestra.run.smithi098:> sudo rpm -qi grub2-tools 2024-08-14T06:43:15.657 INFO:teuthology.orchestra.run.smithi098.stdout:Name : grub2-tools 2024-08-14T06:43:15.657 INFO:teuthology.orchestra.run.smithi098.stdout:Epoch : 1 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Version : 2.06 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Release : 68.el9 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Architecture: x86_64 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Group : Unspecified 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Size : 8264254 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:License : GPLv3+ 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Packager : builder@centos.org 2024-08-14T06:43:15.658 INFO:teuthology.orchestra.run.smithi098.stdout:Vendor : CentOS 2024-08-14T06:43:15.659 INFO:teuthology.orchestra.run.smithi098.stdout:URL : http://www.gnu.org/software/grub/ 2024-08-14T06:43:15.659 INFO:teuthology.orchestra.run.smithi098.stdout:Summary : Support tools for GRUB. 2024-08-14T06:43:15.659 INFO:teuthology.orchestra.run.smithi098.stdout:Description : 2024-08-14T06:43:15.659 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:43:15.659 INFO:teuthology.orchestra.run.smithi098.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-08-14T06:43:15.659 INFO:teuthology.orchestra.run.smithi098.stdout:customizable bootloader with modular architecture. It supports a rich 2024-08-14T06:43:15.659 INFO:teuthology.orchestra.run.smithi098.stdout:variety of kernel formats, file systems, computer architectures and 2024-08-14T06:43:15.659 INFO:teuthology.orchestra.run.smithi098.stdout:hardware devices. 2024-08-14T06:43:15.659 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:43:15.659 INFO:teuthology.orchestra.run.smithi098.stdout:This subpackage provides tools for support of all platforms. 2024-08-14T06:43:15.660 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-08-14T06:43:15.660 INFO:teuthology.task.kernel:Updating grub on smithi098 to boot 5.14.0-480.el9.x86_64 2024-08-14T06:43:15.660 DEBUG:teuthology.orchestra.run.smithi098:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-08-14T06:43:15.785 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-14T06:43:15.786 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:43:15.786 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-08-14T06:43:15.786 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-5.14.0-480.el9.x86_64 2024-08-14T06:43:15.786 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-core-5.14.0-480.el9.x86_64 2024-08-14T06:43:15.786 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-modules-5.14.0-480.el9.x86_64 2024-08-14T06:43:15.786 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-modules-core-5.14.0-480.el9.x86_64 2024-08-14T06:43:15.786 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:43:15.786 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-08-14T06:43:16.247 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q kernel | sort -rV | head -n 1 2024-08-14T06:43:16.278 INFO:teuthology.orchestra.run.smithi098.stderr:Generating grub configuration file ... 2024-08-14T06:43:16.330 INFO:teuthology.orchestra.run.smithi191.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-14T06:43:16.330 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-14T06:43:16.330 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-480.el9.x86_64 2024-08-14T06:43:16.330 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-08-14T06:43:16.330 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-08-14T06:43:16.330 INFO:teuthology.task.kernel:Installing distro kernel on client.1... 2024-08-14T06:43:16.330 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi191.front.sepia.ceph.com, path=None, version=distro) 2024-08-14T06:43:16.330 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q kernel | sort -rV | head -n 1 2024-08-14T06:43:16.387 INFO:teuthology.orchestra.run.smithi191.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-14T06:43:16.387 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-14T06:43:16.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo rpm -qi grub2-tools 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:Name : grub2-tools 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:Epoch : 1 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:Version : 2.06 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:Release : 68.el9 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:Architecture: x86_64 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:Group : Unspecified 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:Size : 8264254 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:License : GPLv3+ 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-08-14T06:43:16.418 INFO:teuthology.orchestra.run.smithi191.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:Packager : builder@centos.org 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:Vendor : CentOS 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:URL : http://www.gnu.org/software/grub/ 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:Summary : Support tools for GRUB. 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:Description : 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:customizable bootloader with modular architecture. It supports a rich 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:variety of kernel formats, file systems, computer architectures and 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout:hardware devices. 2024-08-14T06:43:16.419 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:43:16.420 INFO:teuthology.orchestra.run.smithi191.stdout:This subpackage provides tools for support of all platforms. 2024-08-14T06:43:16.420 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-08-14T06:43:16.421 INFO:teuthology.task.kernel:Updating grub on smithi191 to boot 5.14.0-480.el9.x86_64 2024-08-14T06:43:16.421 DEBUG:teuthology.orchestra.run.smithi191:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-08-14T06:43:17.053 INFO:teuthology.orchestra.run.smithi191.stderr:Generating grub configuration file ... 2024-08-14T06:43:17.635 INFO:teuthology.orchestra.run.smithi098.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-08-14T06:43:17.662 INFO:teuthology.orchestra.run.smithi098.stderr:done 2024-08-14T06:43:17.664 DEBUG:teuthology.orchestra.run.smithi098:> mktemp 2024-08-14T06:43:17.679 INFO:teuthology.orchestra.run.smithi098.stdout:/tmp/tmp.KVxUSYJEIa 2024-08-14T06:43:17.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.KVxUSYJEIa 2024-08-14T06:43:17.744 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 0666 /tmp/tmp.KVxUSYJEIa 2024-08-14T06:43:17.879 DEBUG:teuthology.orchestra.remote:smithi098:/tmp/tmp.KVxUSYJEIa is 6KB 2024-08-14T06:43:17.927 DEBUG:teuthology.orchestra.run.smithi098:> rm -fr /tmp/tmp.KVxUSYJEIa 2024-08-14T06:43:17.941 DEBUG:teuthology.orchestra.run.smithi098:> sudo /bin/ls /boot/loader/entries || true 2024-08-14T06:43:18.002 INFO:teuthology.orchestra.run.smithi098.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-08-14T06:43:18.002 INFO:teuthology.orchestra.run.smithi098.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-08-14T06:43:18.002 INFO:teuthology.orchestra.run.smithi098.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-08-14T06:43:18.002 INFO:teuthology.orchestra.run.smithi098.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64.conf 2024-08-14T06:43:18.008 DEBUG:teuthology.orchestra.run.smithi098:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64 2024-08-14T06:43:18.154 DEBUG:teuthology.orchestra.run.smithi098:> sudo shutdown -r now 2024-08-14T06:43:18.512 INFO:teuthology.orchestra.run.smithi191.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-08-14T06:43:18.537 INFO:teuthology.orchestra.run.smithi191.stderr:done 2024-08-14T06:43:18.542 DEBUG:teuthology.orchestra.run.smithi191:> mktemp 2024-08-14T06:43:18.557 INFO:teuthology.orchestra.run.smithi191.stdout:/tmp/tmp.kfu2AtqHuU 2024-08-14T06:43:18.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.kfu2AtqHuU 2024-08-14T06:43:18.637 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 0666 /tmp/tmp.kfu2AtqHuU 2024-08-14T06:43:18.772 DEBUG:teuthology.orchestra.remote:smithi191:/tmp/tmp.kfu2AtqHuU is 6KB 2024-08-14T06:43:18.822 DEBUG:teuthology.orchestra.run.smithi191:> rm -fr /tmp/tmp.kfu2AtqHuU 2024-08-14T06:43:18.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo /bin/ls /boot/loader/entries || true 2024-08-14T06:43:18.897 INFO:teuthology.orchestra.run.smithi191.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-08-14T06:43:18.897 INFO:teuthology.orchestra.run.smithi191.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-08-14T06:43:18.898 INFO:teuthology.orchestra.run.smithi191.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-08-14T06:43:18.898 INFO:teuthology.orchestra.run.smithi191.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64.conf 2024-08-14T06:43:18.899 DEBUG:teuthology.orchestra.run.smithi191:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64 2024-08-14T06:43:19.049 DEBUG:teuthology.orchestra.run.smithi191:> sudo shutdown -r now 2024-08-14T06:43:48.186 INFO:teuthology.task.kernel:Checking client client.0 for new kernel version... 2024-08-14T06:43:48.187 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2024-08-14T06:43:48.188 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-14T06:43:49.053 INFO:teuthology.task.kernel:Checking client client.1 for new kernel version... 2024-08-14T06:43:49.053 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-08-14T06:43:49.054 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-14T06:44:07.522 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.191 2024-08-14T06:44:16.525 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-08-14T06:44:16.526 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-14T06:44:19.614 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.191 2024-08-14T06:44:22.878 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.98 2024-08-14T06:44:31.617 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-08-14T06:44:31.617 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-14T06:44:31.880 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2024-08-14T06:44:31.880 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-14T06:44:50.271 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.98 2024-08-14T06:45:02.283 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2024-08-14T06:45:02.284 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-14T06:45:02.678 DEBUG:teuthology.orchestra.run.smithi098:> true 2024-08-14T06:45:03.081 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi098.front.sepia.ceph.com' 2024-08-14T06:45:03.081 INFO:teuthology.task.kernel:Checking kernel version of client.0, want "5.14.0-480.el9.x86_64"... 2024-08-14T06:45:03.081 DEBUG:teuthology.orchestra.run.smithi098:> uname -r 2024-08-14T06:45:03.095 INFO:teuthology.orchestra.run.smithi098.stdout:5.14.0-480.el9.x86_64 2024-08-14T06:45:03.095 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-480.el9.x86_64 vs 5.14.0-480.el9.x86_64 2024-08-14T06:45:03.096 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-08-14T06:45:03.096 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-08-14T06:45:03.471 DEBUG:teuthology.orchestra.run.smithi191:> true 2024-08-14T06:45:03.872 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-08-14T06:45:03.873 INFO:teuthology.task.kernel:Checking kernel version of client.1, want "5.14.0-480.el9.x86_64"... 2024-08-14T06:45:03.873 DEBUG:teuthology.orchestra.run.smithi191:> uname -r 2024-08-14T06:45:03.887 INFO:teuthology.orchestra.run.smithi191.stdout:5.14.0-480.el9.x86_64 2024-08-14T06:45:03.887 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-480.el9.x86_64 vs 5.14.0-480.el9.x86_64 2024-08-14T06:45:03.888 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-08-14T06:45:03.888 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-08-14T06:45:04.096 DEBUG:teuthology.parallel:result is None 2024-08-14T06:45:04.888 DEBUG:teuthology.parallel:result is None 2024-08-14T06:45:04.889 INFO:teuthology.run_tasks:Running task internal.base... 2024-08-14T06:45:04.896 INFO:teuthology.task.internal:Creating test directory... 2024-08-14T06:45:04.896 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-14T06:45:04.899 DEBUG:teuthology.orchestra.run.smithi191:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-14T06:45:04.914 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-08-14T06:45:04.920 INFO:teuthology.run_tasks:Running task internal.archive... 2024-08-14T06:45:04.931 INFO:teuthology.task.internal:Creating archive directory... 2024-08-14T06:45:04.931 DEBUG:teuthology.orchestra.run.smithi098:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-14T06:45:04.954 DEBUG:teuthology.orchestra.run.smithi191:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-14T06:45:04.982 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-08-14T06:45:04.989 INFO:teuthology.task.internal:Enabling coredump saving... 2024-08-14T06:45:04.989 DEBUG:teuthology.orchestra.run.smithi098:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-14T06:45:05.010 DEBUG:teuthology.orchestra.run.smithi191:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-14T06:45:05.089 INFO:teuthology.orchestra.run.smithi191.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-14T06:45:05.096 INFO:teuthology.orchestra.run.smithi098.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-14T06:45:05.121 INFO:teuthology.orchestra.run.smithi098.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-14T06:45:05.130 INFO:teuthology.orchestra.run.smithi191.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-14T06:45:05.132 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-08-14T06:45:05.140 INFO:teuthology.task.internal:Configuring sudo... 2024-08-14T06:45:05.140 DEBUG:teuthology.orchestra.run.smithi098:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-14T06:45:05.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-14T06:45:05.167 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-08-14T06:45:05.176 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-08-14T06:45:05.176 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-14T06:45:05.208 DEBUG:teuthology.orchestra.run.smithi191:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-14T06:45:05.227 DEBUG:teuthology.orchestra.run.smithi098:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-14T06:45:05.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-14T06:45:05.353 DEBUG:teuthology.orchestra.run.smithi098:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-14T06:45:05.443 DEBUG:teuthology.orchestra.run.smithi098:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-14T06:45:05.475 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:45:05.476 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-14T06:45:05.552 DEBUG:teuthology.orchestra.run.smithi191:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-14T06:45:05.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-14T06:45:05.649 DEBUG:teuthology.orchestra.run.smithi191:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-14T06:45:05.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-14T06:45:05.777 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:45:05.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-14T06:45:05.845 DEBUG:teuthology.orchestra.run.smithi098:> sudo service rsyslog restart 2024-08-14T06:45:05.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo service rsyslog restart 2024-08-14T06:45:05.891 INFO:teuthology.orchestra.run.smithi098.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-14T06:45:05.924 INFO:teuthology.orchestra.run.smithi191.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-14T06:45:06.307 INFO:teuthology.run_tasks:Running task internal.timer... 2024-08-14T06:45:06.322 INFO:teuthology.task.internal:Starting timer... 2024-08-14T06:45:06.322 INFO:teuthology.run_tasks:Running task pcp... 2024-08-14T06:45:06.334 INFO:teuthology.run_tasks:Running task selinux... 2024-08-14T06:45:06.343 DEBUG:teuthology.orchestra.run.smithi098:> sudo service auditd rotate 2024-08-14T06:45:06.387 INFO:teuthology.orchestra.run.smithi098.stdout:Rotating logs: 2024-08-14T06:45:06.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo service auditd rotate 2024-08-14T06:45:06.507 INFO:teuthology.orchestra.run.smithi191.stdout:Rotating logs: 2024-08-14T06:45:06.511 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-08-14T06:45:06.511 DEBUG:teuthology.orchestra.run.smithi098:> /usr/sbin/getenforce 2024-08-14T06:45:06.532 INFO:teuthology.orchestra.run.smithi098.stdout:Permissive 2024-08-14T06:45:06.532 DEBUG:teuthology.orchestra.run.smithi191:> /usr/sbin/getenforce 2024-08-14T06:45:06.583 INFO:teuthology.orchestra.run.smithi191.stdout:Permissive 2024-08-14T06:45:06.584 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi098.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi191.front.sepia.ceph.com': 'permissive'} 2024-08-14T06:45:06.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-08-14T06:45:06.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-14T06:45:06.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-08-14T06:45:06.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-14T06:45:06.691 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-08-14T06:45:06.691 DEBUG:teuthology.orchestra.run.smithi098:> sudo /usr/sbin/setenforce permissive 2024-08-14T06:45:06.867 DEBUG:teuthology.orchestra.run.smithi191:> sudo /usr/sbin/setenforce permissive 2024-08-14T06:45:07.231 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-08-14T06:45:07.240 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-08-14T06:45:07.251 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-08-14T06:45:07.252 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi098.front.sepia.ceph.com,smithi191.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-08-14T06:49:39.646 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi098.front.sepia.ceph.com'), Remote(name='ubuntu@smithi191.front.sepia.ceph.com')] 2024-08-14T06:49:39.648 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2024-08-14T06:49:39.649 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-14T06:49:39.736 DEBUG:teuthology.orchestra.run.smithi098:> true 2024-08-14T06:49:39.777 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi098.front.sepia.ceph.com' 2024-08-14T06:49:39.777 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-08-14T06:49:39.777 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-14T06:49:39.862 DEBUG:teuthology.orchestra.run.smithi191:> true 2024-08-14T06:49:39.942 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-08-14T06:49:39.942 INFO:teuthology.run_tasks:Running task clock... 2024-08-14T06:49:39.954 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-08-14T06:49:39.954 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-14T06:49:39.955 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-14T06:49:39.957 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-14T06:49:39.958 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-14T06:49:39.985 INFO:teuthology.orchestra.run.smithi098.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-14T06:49:39.999 INFO:teuthology.orchestra.run.smithi098.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-14T06:49:40.012 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-14T06:49:40.023 INFO:teuthology.orchestra.run.smithi098.stderr:sudo: ntpd: command not found 2024-08-14T06:49:40.027 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-14T06:49:40.034 INFO:teuthology.orchestra.run.smithi098.stdout:506 Cannot talk to daemon 2024-08-14T06:49:40.047 INFO:teuthology.orchestra.run.smithi098.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-14T06:49:40.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: ntpd: command not found 2024-08-14T06:49:40.061 INFO:teuthology.orchestra.run.smithi098.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-14T06:49:40.062 INFO:teuthology.orchestra.run.smithi191.stdout:506 Cannot talk to daemon 2024-08-14T06:49:40.077 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-14T06:49:40.091 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-14T06:49:40.115 INFO:teuthology.orchestra.run.smithi098.stderr:bash: line 1: ntpq: command not found 2024-08-14T06:49:40.119 INFO:teuthology.orchestra.run.smithi098.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-14T06:49:40.119 INFO:teuthology.orchestra.run.smithi098.stdout:=============================================================================== 2024-08-14T06:49:40.119 INFO:teuthology.orchestra.run.smithi098.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-14T06:49:40.119 INFO:teuthology.orchestra.run.smithi098.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-14T06:49:40.119 INFO:teuthology.orchestra.run.smithi098.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-14T06:49:40.119 INFO:teuthology.orchestra.run.smithi098.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-14T06:49:40.156 INFO:teuthology.orchestra.run.smithi191.stderr:bash: line 1: ntpq: command not found 2024-08-14T06:49:40.159 INFO:teuthology.orchestra.run.smithi191.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-14T06:49:40.159 INFO:teuthology.orchestra.run.smithi191.stdout:=============================================================================== 2024-08-14T06:49:40.159 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-14T06:49:40.160 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-14T06:49:40.160 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-14T06:49:40.160 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-14T06:49:40.161 INFO:teuthology.run_tasks:Running task install... 2024-08-14T06:49:40.170 DEBUG:teuthology.task.install:project ceph 2024-08-14T06:49:40.170 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3'}} 2024-08-14T06:49:40.170 DEBUG:teuthology.task.install:config {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3'} 2024-08-14T06:49:40.170 INFO:teuthology.task.install:Using flavor: default 2024-08-14T06:49:40.176 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-08-14T06:49:40.176 INFO:teuthology.task.install:extra packages: {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']} 2024-08-14T06:49:40.177 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3', 'tag': None, 'wait_for_package': False} 2024-08-14T06:49:40.177 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=488c4d2c8097ced3f3bab85cfc6687562b3affe3 2024-08-14T06:49:40.179 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3', 'tag': None, 'wait_for_package': False} 2024-08-14T06:49:40.179 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=488c4d2c8097ced3f3bab85cfc6687562b3affe3 2024-08-14T06:49:40.331 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240813.125846-debug/488c4d2c8097ced3f3bab85cfc6687562b3affe3/centos/9/flavors/default/ 2024-08-14T06:49:40.331 INFO:teuthology.task.install.rpm:Package version is 19.0.0-5779.g488c4d2c 2024-08-14T06:49:40.348 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240813.125846-debug/488c4d2c8097ced3f3bab85cfc6687562b3affe3/centos/9/flavors/default/ 2024-08-14T06:49:40.348 INFO:teuthology.task.install.rpm:Package version is 19.0.0-5779.g488c4d2c 2024-08-14T06:49:40.451 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240813.125846-debug/488c4d2c8097ced3f3bab85cfc6687562b3affe3/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240813.125846-debug/488c4d2c8097ced3f3bab85cfc6687562b3affe3/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240813.125846-debug/488c4d2c8097ced3f3bab85cfc6687562b3affe3/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-14T06:49:40.452 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:49:40.452 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-14T06:49:40.462 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240813.125846-debug/488c4d2c8097ced3f3bab85cfc6687562b3affe3/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240813.125846-debug/488c4d2c8097ced3f3bab85cfc6687562b3affe3/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240813.125846-debug/488c4d2c8097ced3f3bab85cfc6687562b3affe3/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-14T06:49:40.462 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:49:40.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-14T06:49:40.478 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-08-14T06:49:40.478 DEBUG:teuthology.orchestra.run.smithi098:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/488c4d2c8097ced3f3bab85cfc6687562b3affe3/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-14T06:49:40.489 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-08-14T06:49:40.489 DEBUG:teuthology.orchestra.run.smithi191:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/488c4d2c8097ced3f3bab85cfc6687562b3affe3/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-14T06:49:40.546 DEBUG:teuthology.orchestra.run.smithi098:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-14T06:49:40.555 DEBUG:teuthology.orchestra.run.smithi191:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-14T06:49:40.622 DEBUG:teuthology.orchestra.run.smithi098:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-14T06:49:40.630 DEBUG:teuthology.orchestra.run.smithi191:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-14T06:49:40.686 INFO:teuthology.orchestra.run.smithi098.stdout:check_obsoletes = 1 2024-08-14T06:49:40.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum clean all 2024-08-14T06:49:40.695 INFO:teuthology.orchestra.run.smithi191.stdout:check_obsoletes = 1 2024-08-14T06:49:40.696 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum clean all 2024-08-14T06:49:40.923 INFO:teuthology.orchestra.run.smithi191.stdout:45 files removed 2024-08-14T06:49:40.925 INFO:teuthology.orchestra.run.smithi098.stdout:45 files removed 2024-08-14T06:49:40.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-08-14T06:49:40.955 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-08-14T06:49:41.537 INFO:teuthology.orchestra.run.smithi191.stdout:ceph packages for x86_64 272 kB/s | 84 kB 00:00 2024-08-14T06:49:41.550 INFO:teuthology.orchestra.run.smithi098.stdout:ceph packages for x86_64 263 kB/s | 84 kB 00:00 2024-08-14T06:49:41.887 INFO:teuthology.orchestra.run.smithi191.stdout:ceph noarch packages 80 kB/s | 24 kB 00:00 2024-08-14T06:49:41.892 INFO:teuthology.orchestra.run.smithi098.stdout:ceph noarch packages 82 kB/s | 24 kB 00:00 2024-08-14T06:49:42.229 INFO:teuthology.orchestra.run.smithi191.stdout:ceph source packages 6.3 kB/s | 1.9 kB 00:00 2024-08-14T06:49:42.243 INFO:teuthology.orchestra.run.smithi098.stdout:ceph source packages 6.2 kB/s | 1.9 kB 00:00 2024-08-14T06:49:42.796 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.2 MB 00:00 2024-08-14T06:49:45.194 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 9 - BaseOS 2.8 MB/s | 8.2 MB 00:02 2024-08-14T06:49:46.939 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - AppStream 6.6 MB/s | 20 MB 00:03 2024-08-14T06:49:50.944 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 9 - AppStream 4.3 MB/s | 20 MB 00:04 2024-08-14T06:49:51.465 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - CRB 8.3 MB/s | 6.6 MB 00:00 2024-08-14T06:49:53.215 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - Extras packages 44 kB/s | 18 kB 00:00 2024-08-14T06:49:54.486 INFO:teuthology.orchestra.run.smithi191.stdout:Extra Packages for Enterprise Linux 20 MB/s | 22 MB 00:01 2024-08-14T06:49:56.645 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 9 - CRB 3.3 MB/s | 6.6 MB 00:01 2024-08-14T06:49:58.112 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 9 - Extras packages 83 kB/s | 18 kB 00:00 2024-08-14T06:49:58.950 INFO:teuthology.orchestra.run.smithi098.stdout:Extra Packages for Enterprise Linux 31 MB/s | 22 MB 00:00 2024-08-14T06:49:59.892 INFO:teuthology.orchestra.run.smithi191.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-08-14T06:50:02.196 INFO:teuthology.orchestra.run.smithi191.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-08-14T06:50:02.196 INFO:teuthology.orchestra.run.smithi191.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-08-14T06:50:02.208 INFO:teuthology.orchestra.run.smithi191.stdout:Package NetworkManager-1:1.47.4-1.el9.x86_64 is already installed. 2024-08-14T06:50:02.208 INFO:teuthology.orchestra.run.smithi191.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-08-14T06:50:02.209 INFO:teuthology.orchestra.run.smithi191.stdout:Package util-linux-2.37.4-18.el9.x86_64 is already installed. 2024-08-14T06:50:02.210 INFO:teuthology.orchestra.run.smithi191.stdout:Package libacl-devel-2.3.1-4.el9.x86_64 is already installed. 2024-08-14T06:50:02.212 INFO:teuthology.orchestra.run.smithi191.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-08-14T06:50:02.214 INFO:teuthology.orchestra.run.smithi191.stdout:Package xfsdump-3.1.12-4.el9.x86_64 is already installed. 2024-08-14T06:50:02.215 INFO:teuthology.orchestra.run.smithi191.stdout:Package xfsprogs-6.3.0-1.el9.x86_64 is already installed. 2024-08-14T06:50:02.277 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout:======================================================================================= 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout:======================================================================================= 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout: bison x86_64 3.7.4-5.el9 appstream 943 k 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout: ceph x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 6.6 k 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-base x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 5.5 M 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-fuse x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 873 k 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 148 k 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 1.5 M 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-cephadm noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 150 k 2024-08-14T06:50:02.288 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-dashboard noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 14 M 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 7.4 M 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-rook noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 49 k 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-radosgw x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 10 M 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-test x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 44 M 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-volume noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 276 k 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: cephadm noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 773 k 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: cephfs-mirror x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 225 k 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: cephfs-shell x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 27 k 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: cephfs-top noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 18 k 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: elfutils-libelf-devel x86_64 0.191-4.el9 appstream 22 k 2024-08-14T06:50:02.289 INFO:teuthology.orchestra.run.smithi191.stdout: flex x86_64 2.6.4-9.el9 appstream 313 k 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: libaio-devel x86_64 0.3.111-13.el9 appstream 11 k 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs-devel x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 32 k 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs2 x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 765 k 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: librados-devel x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 127 k 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: libtool x86_64 2.4.6-46.el9 appstream 581 k 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: openssl-devel x86_64 1:3.2.2-2.el9 appstream 4.4 M 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: postgresql x86_64 13.14-1.el9 appstream 1.6 M 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: postgresql-contrib x86_64 13.14-1.el9 appstream 911 k 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: postgresql-server x86_64 13.14-1.el9 appstream 5.8 M 2024-08-14T06:50:02.290 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cephfs x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 188 k 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rados x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 370 k 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rbd x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 356 k 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rgw x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 116 k 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-fuse x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 86 k 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-mirror x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 3.1 M 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-nbd x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 170 k 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: xfsprogs-devel x86_64 6.4.0-3.el9 appstream 96 k 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout:Upgrading: 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: NetworkManager x86_64 1:1.48.2-2.el9 baseos 2.3 M 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: NetworkManager-libnm x86_64 1:1.48.2-2.el9 baseos 1.8 M 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: NetworkManager-team x86_64 1:1.48.2-2.el9 baseos 41 k 2024-08-14T06:50:02.291 INFO:teuthology.orchestra.run.smithi191.stdout: NetworkManager-tui x86_64 1:1.48.2-2.el9 baseos 247 k 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: elfutils-debuginfod-client x86_64 0.191-4.el9 baseos 37 k 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: elfutils-libelf x86_64 0.191-4.el9 baseos 207 k 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: elfutils-libs x86_64 0.191-4.el9 baseos 260 k 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: librados2 x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 3.4 M 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: librbd1 x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 3.1 M 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: openssl x86_64 1:3.2.2-2.el9 baseos 1.4 M 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: openssl-libs x86_64 1:3.2.2-2.el9 baseos 2.4 M 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: xfsprogs x86_64 6.4.0-3.el9 baseos 1.1 M 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: automake noarch 1.16.2-8.el9 appstream 689 k 2024-08-14T06:50:02.292 INFO:teuthology.orchestra.run.smithi191.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-common x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 20 M 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-grafana-dashboards noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 34 k 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mds x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 2.3 M 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-modules-core noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 284 k 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mon x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 4.9 M 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-osd x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 18 M 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-prometheus-alerts noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 16 k 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-selinux x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 25 k 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-14T06:50:02.293 INFO:teuthology.orchestra.run.smithi191.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: libcephsqlite x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 158 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: libradosstriper1 x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 504 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-14T06:50:02.294 INFO:teuthology.orchestra.run.smithi191.stdout: librgw2 x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 5.3 M 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: libzstd-devel x86_64 1.5.1-2.el9 appstream 47 k 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: postgresql-private-libs x86_64 13.14-1.el9 appstream 133 k 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-08-14T06:50:02.295 INFO:teuthology.orchestra.run.smithi191.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-argparse x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 45 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-common x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 138 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-colorama noarch 0.4.6-3.el9 appstream 61 k 2024-08-14T06:50:02.296 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-google-auth noarch 1:2.32.0-1.el9 epel 216 k 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-14T06:50:02.297 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-14T06:50:02.298 INFO:teuthology.orchestra.run.smithi191.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-14T06:50:02.299 INFO:teuthology.orchestra.run.smithi191.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: python3-wcwidth noarch 0.2.5-8.el9 appstream 44 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-08-14T06:50:02.300 INFO:teuthology.orchestra.run.smithi191.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout: uuid x86_64 1.6.2-55.el9 appstream 58 k 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout: zlib-devel x86_64 1.2.11-41.el9 appstream 45 k 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout:Installing weak dependencies: 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout:======================================================================================= 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout:Install 148 Packages 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout:Upgrade 12 Packages 2024-08-14T06:50:02.301 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:02.302 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 233 M 2024-08-14T06:50:02.302 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-08-14T06:50:02.821 INFO:teuthology.orchestra.run.smithi191.stdout:(1/160): ceph-19.0.0-5779.g488c4d2c.el9.x86_64. 35 kB/s | 6.6 kB 00:00 2024-08-14T06:50:03.121 INFO:teuthology.orchestra.run.smithi191.stdout:(2/160): ceph-base-19.0.0-5779.g488c4d2c.el9.x8 11 MB/s | 5.5 MB 00:00 2024-08-14T06:50:03.155 INFO:teuthology.orchestra.run.smithi191.stdout:(3/160): ceph-fuse-19.0.0-5779.g488c4d2c.el9.x8 2.6 MB/s | 873 kB 00:00 2024-08-14T06:50:03.180 INFO:teuthology.orchestra.run.smithi191.stdout:(4/160): ceph-immutable-object-cache-19.0.0-577 2.5 MB/s | 148 kB 00:00 2024-08-14T06:50:03.272 INFO:teuthology.orchestra.run.smithi191.stdout:(5/160): ceph-mgr-19.0.0-5779.g488c4d2c.el9.x86 16 MB/s | 1.5 MB 00:00 2024-08-14T06:50:03.539 INFO:teuthology.orchestra.run.smithi191.stdout:(6/160): ceph-mon-19.0.0-5779.g488c4d2c.el9.x86 18 MB/s | 4.9 MB 00:00 2024-08-14T06:50:03.581 INFO:teuthology.orchestra.run.smithi191.stdout:(7/160): ceph-mds-19.0.0-5779.g488c4d2c.el9.x86 5.3 MB/s | 2.3 MB 00:00 2024-08-14T06:50:03.874 INFO:teuthology.orchestra.run.smithi191.stdout:(8/160): ceph-common-19.0.0-5779.g488c4d2c.el9. 16 MB/s | 20 MB 00:01 2024-08-14T06:50:03.924 INFO:teuthology.orchestra.run.smithi191.stdout:(9/160): ceph-selinux-19.0.0-5779.g488c4d2c.el9 504 kB/s | 25 kB 00:00 2024-08-14T06:50:04.400 INFO:teuthology.orchestra.run.smithi191.stdout:(10/160): ceph-osd-19.0.0-5779.g488c4d2c.el9.x8 21 MB/s | 18 MB 00:00 2024-08-14T06:50:04.406 INFO:teuthology.orchestra.run.smithi098.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-08-14T06:50:04.450 INFO:teuthology.orchestra.run.smithi191.stdout:(11/160): cephfs-mirror-19.0.0-5779.g488c4d2c.e 4.4 MB/s | 225 kB 00:00 2024-08-14T06:50:04.584 INFO:teuthology.orchestra.run.smithi191.stdout:(12/160): ceph-radosgw-19.0.0-5779.g488c4d2c.el 10 MB/s | 10 MB 00:01 2024-08-14T06:50:04.609 INFO:teuthology.orchestra.run.smithi191.stdout:(13/160): cephfs-shell-19.0.0-5779.g488c4d2c.el 170 kB/s | 27 kB 00:00 2024-08-14T06:50:04.634 INFO:teuthology.orchestra.run.smithi191.stdout:(14/160): libcephfs-devel-19.0.0-5779.g488c4d2c 632 kB/s | 32 kB 00:00 2024-08-14T06:50:04.676 INFO:teuthology.orchestra.run.smithi191.stdout:(15/160): libcephfs2-19.0.0-5779.g488c4d2c.el9. 11 MB/s | 765 kB 00:00 2024-08-14T06:50:04.702 INFO:teuthology.orchestra.run.smithi191.stdout:(16/160): libcephsqlite-19.0.0-5779.g488c4d2c.e 2.3 MB/s | 158 kB 00:00 2024-08-14T06:50:04.727 INFO:teuthology.orchestra.run.smithi191.stdout:(17/160): librados-devel-19.0.0-5779.g488c4d2c. 2.4 MB/s | 127 kB 00:00 2024-08-14T06:50:04.770 INFO:teuthology.orchestra.run.smithi191.stdout:(18/160): libradosstriper1-19.0.0-5779.g488c4d2 7.3 MB/s | 504 kB 00:00 2024-08-14T06:50:04.837 INFO:teuthology.orchestra.run.smithi191.stdout:(19/160): python3-ceph-argparse-19.0.0-5779.g48 672 kB/s | 45 kB 00:00 2024-08-14T06:50:04.904 INFO:teuthology.orchestra.run.smithi191.stdout:(20/160): python3-ceph-common-19.0.0-5779.g488c 2.0 MB/s | 138 kB 00:00 2024-08-14T06:50:04.987 INFO:teuthology.orchestra.run.smithi191.stdout:(21/160): librgw2-19.0.0-5779.g488c4d2c.el9.x86 20 MB/s | 5.3 MB 00:00 2024-08-14T06:50:05.013 INFO:teuthology.orchestra.run.smithi191.stdout:(22/160): python3-cephfs-19.0.0-5779.g488c4d2c. 1.7 MB/s | 188 kB 00:00 2024-08-14T06:50:05.046 INFO:teuthology.orchestra.run.smithi191.stdout:(23/160): python3-rados-19.0.0-5779.g488c4d2c.e 6.2 MB/s | 370 kB 00:00 2024-08-14T06:50:05.072 INFO:teuthology.orchestra.run.smithi191.stdout:(24/160): python3-rbd-19.0.0-5779.g488c4d2c.el9 5.9 MB/s | 356 kB 00:00 2024-08-14T06:50:05.097 INFO:teuthology.orchestra.run.smithi191.stdout:(25/160): python3-rgw-19.0.0-5779.g488c4d2c.el9 2.2 MB/s | 116 kB 00:00 2024-08-14T06:50:05.122 INFO:teuthology.orchestra.run.smithi191.stdout:(26/160): rbd-fuse-19.0.0-5779.g488c4d2c.el9.x8 1.7 MB/s | 86 kB 00:00 2024-08-14T06:50:05.173 INFO:teuthology.orchestra.run.smithi191.stdout:(27/160): rbd-nbd-19.0.0-5779.g488c4d2c.el9.x86 3.3 MB/s | 170 kB 00:00 2024-08-14T06:50:05.240 INFO:teuthology.orchestra.run.smithi191.stdout:(28/160): rbd-mirror-19.0.0-5779.g488c4d2c.el9. 22 MB/s | 3.1 MB 00:00 2024-08-14T06:50:05.266 INFO:teuthology.orchestra.run.smithi191.stdout:(29/160): ceph-grafana-dashboards-19.0.0-5779.g 373 kB/s | 34 kB 00:00 2024-08-14T06:50:05.291 INFO:teuthology.orchestra.run.smithi191.stdout:(30/160): ceph-mgr-cephadm-19.0.0-5779.g488c4d2 2.9 MB/s | 150 kB 00:00 2024-08-14T06:50:05.750 INFO:teuthology.orchestra.run.smithi191.stdout:(31/160): ceph-mgr-diskprediction-local-19.0.0- 16 MB/s | 7.4 MB 00:00 2024-08-14T06:50:05.817 INFO:teuthology.orchestra.run.smithi191.stdout:(32/160): ceph-mgr-modules-core-19.0.0-5779.g48 4.2 MB/s | 284 kB 00:00 2024-08-14T06:50:05.892 INFO:teuthology.orchestra.run.smithi191.stdout:(33/160): ceph-mgr-rook-19.0.0-5779.g488c4d2c.e 656 kB/s | 49 kB 00:00 2024-08-14T06:50:06.084 INFO:teuthology.orchestra.run.smithi191.stdout:(34/160): ceph-mgr-dashboard-19.0.0-5779.g488c4 17 MB/s | 14 MB 00:00 2024-08-14T06:50:06.510 INFO:teuthology.orchestra.run.smithi191.stdout:(35/160): ceph-test-19.0.0-5779.g488c4d2c.el9.x 17 MB/s | 44 MB 00:02 2024-08-14T06:50:06.535 INFO:teuthology.orchestra.run.smithi191.stdout:(36/160): ceph-prometheus-alerts-19.0.0-5779.g4 26 kB/s | 16 kB 00:00 2024-08-14T06:50:06.560 INFO:teuthology.orchestra.run.smithi191.stdout:(37/160): ceph-volume-19.0.0-5779.g488c4d2c.el9 580 kB/s | 276 kB 00:00 2024-08-14T06:50:06.586 INFO:teuthology.orchestra.run.smithi191.stdout:(38/160): cephfs-top-19.0.0-5779.g488c4d2c.el9. 364 kB/s | 18 kB 00:00 2024-08-14T06:50:06.619 INFO:teuthology.orchestra.run.smithi191.stdout:(39/160): cephadm-19.0.0-5779.g488c4d2c.el9.noa 9.0 MB/s | 773 kB 00:00 2024-08-14T06:50:06.695 INFO:teuthology.orchestra.run.smithi191.stdout:(40/160): fuse-2.9.9-16.el9.x86_64.rpm 594 kB/s | 80 kB 00:00 2024-08-14T06:50:06.724 INFO:teuthology.orchestra.run.smithi098.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-08-14T06:50:06.724 INFO:teuthology.orchestra.run.smithi098.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-08-14T06:50:06.729 INFO:teuthology.orchestra.run.smithi191.stdout:(41/160): libgfortran-11.4.1-3.el9.x86_64.rpm 5.5 MB/s | 803 kB 00:00 2024-08-14T06:50:06.735 INFO:teuthology.orchestra.run.smithi098.stdout:Package NetworkManager-1:1.47.4-1.el9.x86_64 is already installed. 2024-08-14T06:50:06.736 INFO:teuthology.orchestra.run.smithi098.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-08-14T06:50:06.737 INFO:teuthology.orchestra.run.smithi098.stdout:Package util-linux-2.37.4-18.el9.x86_64 is already installed. 2024-08-14T06:50:06.737 INFO:teuthology.orchestra.run.smithi098.stdout:Package libacl-devel-2.3.1-4.el9.x86_64 is already installed. 2024-08-14T06:50:06.739 INFO:teuthology.orchestra.run.smithi098.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-08-14T06:50:06.741 INFO:teuthology.orchestra.run.smithi098.stdout:Package xfsdump-3.1.12-4.el9.x86_64 is already installed. 2024-08-14T06:50:06.742 INFO:teuthology.orchestra.run.smithi098.stdout:Package xfsprogs-6.3.0-1.el9.x86_64 is already installed. 2024-08-14T06:50:06.754 INFO:teuthology.orchestra.run.smithi191.stdout:(42/160): libquadmath-11.4.1-3.el9.x86_64.rpm 1.4 MB/s | 195 kB 00:00 2024-08-14T06:50:06.780 INFO:teuthology.orchestra.run.smithi191.stdout:(43/160): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.9 MB/s | 253 kB 00:00 2024-08-14T06:50:06.805 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2024-08-14T06:50:06.806 INFO:teuthology.orchestra.run.smithi191.stdout:(44/160): python3-ply-3.11-14.el9.noarch.rpm 2.1 MB/s | 106 kB 00:00 2024-08-14T06:50:06.825 INFO:teuthology.orchestra.run.smithi098.stdout:======================================================================================= 2024-08-14T06:50:06.826 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2024-08-14T06:50:06.826 INFO:teuthology.orchestra.run.smithi098.stdout:======================================================================================= 2024-08-14T06:50:06.826 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2024-08-14T06:50:06.826 INFO:teuthology.orchestra.run.smithi098.stdout: bison x86_64 3.7.4-5.el9 appstream 943 k 2024-08-14T06:50:06.826 INFO:teuthology.orchestra.run.smithi098.stdout: ceph x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 6.6 k 2024-08-14T06:50:06.826 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-base x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 5.5 M 2024-08-14T06:50:06.826 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-fuse x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 873 k 2024-08-14T06:50:06.826 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 148 k 2024-08-14T06:50:06.827 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 1.5 M 2024-08-14T06:50:06.827 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-cephadm noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 150 k 2024-08-14T06:50:06.827 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-dashboard noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 14 M 2024-08-14T06:50:06.827 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 7.4 M 2024-08-14T06:50:06.827 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-rook noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 49 k 2024-08-14T06:50:06.827 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-radosgw x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 10 M 2024-08-14T06:50:06.827 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-test x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 44 M 2024-08-14T06:50:06.827 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-volume noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 276 k 2024-08-14T06:50:06.827 INFO:teuthology.orchestra.run.smithi098.stdout: cephadm noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 773 k 2024-08-14T06:50:06.827 INFO:teuthology.orchestra.run.smithi098.stdout: cephfs-mirror x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 225 k 2024-08-14T06:50:06.828 INFO:teuthology.orchestra.run.smithi098.stdout: cephfs-shell x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 27 k 2024-08-14T06:50:06.828 INFO:teuthology.orchestra.run.smithi098.stdout: cephfs-top noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 18 k 2024-08-14T06:50:06.828 INFO:teuthology.orchestra.run.smithi098.stdout: elfutils-libelf-devel x86_64 0.191-4.el9 appstream 22 k 2024-08-14T06:50:06.828 INFO:teuthology.orchestra.run.smithi098.stdout: flex x86_64 2.6.4-9.el9 appstream 313 k 2024-08-14T06:50:06.828 INFO:teuthology.orchestra.run.smithi098.stdout: libaio-devel x86_64 0.3.111-13.el9 appstream 11 k 2024-08-14T06:50:06.828 INFO:teuthology.orchestra.run.smithi098.stdout: libcephfs-devel x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 32 k 2024-08-14T06:50:06.828 INFO:teuthology.orchestra.run.smithi098.stdout: libcephfs2 x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 765 k 2024-08-14T06:50:06.828 INFO:teuthology.orchestra.run.smithi098.stdout: librados-devel x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 127 k 2024-08-14T06:50:06.828 INFO:teuthology.orchestra.run.smithi098.stdout: libtool x86_64 2.4.6-46.el9 appstream 581 k 2024-08-14T06:50:06.828 INFO:teuthology.orchestra.run.smithi098.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-08-14T06:50:06.829 INFO:teuthology.orchestra.run.smithi098.stdout: openssl-devel x86_64 1:3.2.2-2.el9 appstream 4.4 M 2024-08-14T06:50:06.829 INFO:teuthology.orchestra.run.smithi098.stdout: postgresql x86_64 13.14-1.el9 appstream 1.6 M 2024-08-14T06:50:06.829 INFO:teuthology.orchestra.run.smithi098.stdout: postgresql-contrib x86_64 13.14-1.el9 appstream 911 k 2024-08-14T06:50:06.829 INFO:teuthology.orchestra.run.smithi098.stdout: postgresql-server x86_64 13.14-1.el9 appstream 5.8 M 2024-08-14T06:50:06.829 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cephfs x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 188 k 2024-08-14T06:50:06.829 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rados x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 370 k 2024-08-14T06:50:06.829 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rbd x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 356 k 2024-08-14T06:50:06.829 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rgw x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 116 k 2024-08-14T06:50:06.830 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-fuse x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 86 k 2024-08-14T06:50:06.830 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-mirror x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 3.1 M 2024-08-14T06:50:06.830 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-nbd x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 170 k 2024-08-14T06:50:06.830 INFO:teuthology.orchestra.run.smithi098.stdout: xfsprogs-devel x86_64 6.4.0-3.el9 appstream 96 k 2024-08-14T06:50:06.830 INFO:teuthology.orchestra.run.smithi098.stdout:Upgrading: 2024-08-14T06:50:06.830 INFO:teuthology.orchestra.run.smithi098.stdout: NetworkManager x86_64 1:1.48.2-2.el9 baseos 2.3 M 2024-08-14T06:50:06.830 INFO:teuthology.orchestra.run.smithi098.stdout: NetworkManager-libnm x86_64 1:1.48.2-2.el9 baseos 1.8 M 2024-08-14T06:50:06.830 INFO:teuthology.orchestra.run.smithi098.stdout: NetworkManager-team x86_64 1:1.48.2-2.el9 baseos 41 k 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout: NetworkManager-tui x86_64 1:1.48.2-2.el9 baseos 247 k 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout: elfutils-debuginfod-client x86_64 0.191-4.el9 baseos 37 k 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout: elfutils-libelf x86_64 0.191-4.el9 baseos 207 k 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout: elfutils-libs x86_64 0.191-4.el9 baseos 260 k 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout: librados2 x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 3.4 M 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout: librbd1 x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 3.1 M 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout: openssl x86_64 1:3.2.2-2.el9 baseos 1.4 M 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout: openssl-libs x86_64 1:3.2.2-2.el9 baseos 2.4 M 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout: xfsprogs x86_64 6.4.0-3.el9 baseos 1.1 M 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2024-08-14T06:50:06.831 INFO:teuthology.orchestra.run.smithi098.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: automake noarch 1.16.2-8.el9 appstream 689 k 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-common x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 20 M 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-grafana-dashboards noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 34 k 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mds x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 2.3 M 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-modules-core noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 284 k 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mon x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 4.9 M 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-osd x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 18 M 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-prometheus-alerts noarch 2:19.0.0-5779.g488c4d2c.el9 ceph-noarch 16 k 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-selinux x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 25 k 2024-08-14T06:50:06.832 INFO:teuthology.orchestra.run.smithi098.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: libcephsqlite x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 158 k 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-14T06:50:06.833 INFO:teuthology.orchestra.run.smithi098.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-14T06:50:06.834 INFO:teuthology.orchestra.run.smithi098.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-08-14T06:50:06.834 INFO:teuthology.orchestra.run.smithi098.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-14T06:50:06.834 INFO:teuthology.orchestra.run.smithi098.stdout: libradosstriper1 x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 504 k 2024-08-14T06:50:06.834 INFO:teuthology.orchestra.run.smithi098.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-14T06:50:06.834 INFO:teuthology.orchestra.run.smithi098.stdout: librgw2 x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 5.3 M 2024-08-14T06:50:06.834 INFO:teuthology.orchestra.run.smithi098.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-14T06:50:06.834 INFO:teuthology.orchestra.run.smithi098.stdout: libzstd-devel x86_64 1.5.1-2.el9 appstream 47 k 2024-08-14T06:50:06.835 INFO:teuthology.orchestra.run.smithi098.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-14T06:50:06.835 INFO:teuthology.orchestra.run.smithi098.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-08-14T06:50:06.835 INFO:teuthology.orchestra.run.smithi098.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-08-14T06:50:06.835 INFO:teuthology.orchestra.run.smithi098.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-08-14T06:50:06.835 INFO:teuthology.orchestra.run.smithi098.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-08-14T06:50:06.835 INFO:teuthology.orchestra.run.smithi098.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-08-14T06:50:06.835 INFO:teuthology.orchestra.run.smithi098.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-08-14T06:50:06.835 INFO:teuthology.orchestra.run.smithi098.stdout: postgresql-private-libs x86_64 13.14-1.el9 appstream 133 k 2024-08-14T06:50:06.836 INFO:teuthology.orchestra.run.smithi098.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-08-14T06:50:06.836 INFO:teuthology.orchestra.run.smithi098.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-08-14T06:50:06.836 INFO:teuthology.orchestra.run.smithi098.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-14T06:50:06.836 INFO:teuthology.orchestra.run.smithi098.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-08-14T06:50:06.836 INFO:teuthology.orchestra.run.smithi098.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-14T06:50:06.836 INFO:teuthology.orchestra.run.smithi098.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-14T06:50:06.836 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-14T06:50:06.836 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ceph-argparse x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 45 k 2024-08-14T06:50:06.837 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ceph-common x86_64 2:19.0.0-5779.g488c4d2c.el9 ceph 138 k 2024-08-14T06:50:06.837 INFO:teuthology.orchestra.run.smithi098.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-14T06:50:06.837 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-14T06:50:06.837 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-14T06:50:06.837 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-14T06:50:06.837 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-08-14T06:50:06.837 INFO:teuthology.orchestra.run.smithi098.stdout: python3-colorama noarch 0.4.6-3.el9 appstream 61 k 2024-08-14T06:50:06.837 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-14T06:50:06.838 INFO:teuthology.orchestra.run.smithi098.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-08-14T06:50:06.838 INFO:teuthology.orchestra.run.smithi098.stdout: python3-google-auth noarch 1:2.32.0-1.el9 epel 216 k 2024-08-14T06:50:06.838 INFO:teuthology.orchestra.run.smithi098.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-08-14T06:50:06.838 INFO:teuthology.orchestra.run.smithi098.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-08-14T06:50:06.838 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-14T06:50:06.838 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-14T06:50:06.838 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-08-14T06:50:06.838 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-14T06:50:06.838 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-14T06:50:06.839 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-14T06:50:06.839 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-14T06:50:06.839 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-14T06:50:06.839 INFO:teuthology.orchestra.run.smithi098.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-14T06:50:06.839 INFO:teuthology.orchestra.run.smithi098.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-14T06:50:06.839 INFO:teuthology.orchestra.run.smithi098.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-14T06:50:06.839 INFO:teuthology.orchestra.run.smithi098.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-14T06:50:06.839 INFO:teuthology.orchestra.run.smithi098.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-14T06:50:06.839 INFO:teuthology.orchestra.run.smithi098.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-14T06:50:06.839 INFO:teuthology.orchestra.run.smithi098.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-14T06:50:06.840 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-wcwidth noarch 0.2.5-8.el9 appstream 44 k 2024-08-14T06:50:06.841 INFO:teuthology.orchestra.run.smithi098.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: uuid x86_64 1.6.2-55.el9 appstream 58 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout: zlib-devel x86_64 1.2.11-41.el9 appstream 45 k 2024-08-14T06:50:06.842 INFO:teuthology.orchestra.run.smithi098.stdout:Installing weak dependencies: 2024-08-14T06:50:06.843 INFO:teuthology.orchestra.run.smithi098.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-08-14T06:50:06.843 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:50:06.843 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2024-08-14T06:50:06.843 INFO:teuthology.orchestra.run.smithi098.stdout:======================================================================================= 2024-08-14T06:50:06.843 INFO:teuthology.orchestra.run.smithi098.stdout:Install 148 Packages 2024-08-14T06:50:06.843 INFO:teuthology.orchestra.run.smithi098.stdout:Upgrade 12 Packages 2024-08-14T06:50:06.843 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:50:06.844 INFO:teuthology.orchestra.run.smithi191.stdout:(45/160): python3-pycparser-2.20-6.el9.noarch.r 2.6 MB/s | 135 kB 00:00 2024-08-14T06:50:06.844 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 233 M 2024-08-14T06:50:06.844 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2024-08-14T06:50:06.872 INFO:teuthology.orchestra.run.smithi191.stdout:(46/160): python3-cryptography-36.0.1-4.el9.x86 8.6 MB/s | 1.2 MB 00:00 2024-08-14T06:50:06.898 INFO:teuthology.orchestra.run.smithi191.stdout:(47/160): python3-requests-2.25.1-8.el9.noarch. 1.3 MB/s | 125 kB 00:00 2024-08-14T06:50:06.923 INFO:teuthology.orchestra.run.smithi191.stdout:(48/160): python3-urllib3-1.26.5-6.el9.noarch.r 2.3 MB/s | 215 kB 00:00 2024-08-14T06:50:07.040 INFO:teuthology.orchestra.run.smithi191.stdout:(49/160): boost-program-options-1.75.0-8.el9.x8 917 kB/s | 107 kB 00:00 2024-08-14T06:50:07.074 INFO:teuthology.orchestra.run.smithi191.stdout:(50/160): automake-1.16.2-8.el9.noarch.rpm 3.3 MB/s | 689 kB 00:00 2024-08-14T06:50:07.099 INFO:teuthology.orchestra.run.smithi191.stdout:(51/160): elfutils-libelf-devel-0.191-4.el9.x86 368 kB/s | 22 kB 00:00 2024-08-14T06:50:07.142 INFO:teuthology.orchestra.run.smithi191.stdout:(52/160): bison-3.7.4-5.el9.x86_64.rpm 3.8 MB/s | 943 kB 00:00 2024-08-14T06:50:07.167 INFO:teuthology.orchestra.run.smithi191.stdout:(53/160): flexiblas-3.0.4-8.el9.x86_64.rpm 488 kB/s | 33 kB 00:00 2024-08-14T06:50:07.192 INFO:teuthology.orchestra.run.smithi191.stdout:(54/160): flex-2.6.4-9.el9.x86_64.rpm 2.6 MB/s | 313 kB 00:00 2024-08-14T06:50:07.218 INFO:teuthology.orchestra.run.smithi191.stdout:(55/160): flexiblas-openblas-openmp-3.0.4-8.el9 360 kB/s | 18 kB 00:00 2024-08-14T06:50:07.287 INFO:teuthology.orchestra.run.smithi191.stdout:(56/160): libaio-devel-0.3.111-13.el9.x86_64.rp 220 kB/s | 11 kB 00:00 2024-08-14T06:50:07.287 INFO:teuthology.orchestra.run.smithi191.stdout:(57/160): libpmemobj-1.12.1-1.el9.x86_64.rpm 3.1 MB/s | 160 kB 00:00 2024-08-14T06:50:07.294 INFO:teuthology.orchestra.run.smithi191.stdout:(58/160): librabbitmq-0.11.0-7.el9.x86_64.rpm 898 kB/s | 45 kB 00:00 2024-08-14T06:50:07.328 INFO:teuthology.orchestra.run.smithi191.stdout:(59/160): librdkafka-1.6.1-102.el9.x86_64.rpm 11 MB/s | 662 kB 00:00 2024-08-14T06:50:07.386 INFO:teuthology.orchestra.run.smithi191.stdout:(60/160): flexiblas-netlib-3.0.4-8.el9.x86_64.r 12 MB/s | 3.0 MB 00:00 2024-08-14T06:50:07.412 INFO:teuthology.orchestra.run.smithi191.stdout:(61/160): libuuid-devel-2.37.4-18.el9.x86_64.rp 303 kB/s | 25 kB 00:00 2024-08-14T06:50:07.437 INFO:teuthology.orchestra.run.smithi191.stdout:(62/160): libzstd-devel-1.5.1-2.el9.x86_64.rpm 942 kB/s | 47 kB 00:00 2024-08-14T06:50:07.463 INFO:teuthology.orchestra.run.smithi191.stdout:(63/160): lttng-ust-2.12.0-6.el9.x86_64.rpm 5.7 MB/s | 292 kB 00:00 2024-08-14T06:50:07.488 INFO:teuthology.orchestra.run.smithi191.stdout:(64/160): libtool-2.4.6-46.el9.x86_64.rpm 2.9 MB/s | 581 kB 00:00 2024-08-14T06:50:07.514 INFO:teuthology.orchestra.run.smithi191.stdout:(65/160): openblas-0.3.21-2.el9.x86_64.rpm 699 kB/s | 35 kB 00:00 2024-08-14T06:50:07.539 INFO:teuthology.orchestra.run.smithi191.stdout:(66/160): lua-5.4.4-4.el9.x86_64.rpm 1.8 MB/s | 188 kB 00:00 2024-08-14T06:50:07.606 INFO:teuthology.orchestra.run.smithi191.stdout:(67/160): postgresql-13.14-1.el9.x86_64.rpm 24 MB/s | 1.6 MB 00:00 2024-08-14T06:50:07.682 INFO:teuthology.orchestra.run.smithi191.stdout:(68/160): openblas-openmp-0.3.21-2.el9.x86_64.r 25 MB/s | 4.7 MB 00:00 2024-08-14T06:50:07.787 INFO:teuthology.orchestra.run.smithi191.stdout:(69/160): openssl-devel-3.2.2-2.el9.x86_64.rpm 16 MB/s | 4.4 MB 00:00 2024-08-14T06:50:07.832 INFO:teuthology.orchestra.run.smithi191.stdout:(70/160): postgresql-private-libs-13.14-1.el9.x 886 kB/s | 133 kB 00:00 2024-08-14T06:50:07.874 INFO:teuthology.orchestra.run.smithi191.stdout:(71/160): postgresql-contrib-13.14-1.el9.x86_64 3.3 MB/s | 911 kB 00:00 2024-08-14T06:50:07.916 INFO:teuthology.orchestra.run.smithi191.stdout:(72/160): protobuf-3.14.0-13.el9.x86_64.rpm 12 MB/s | 1.0 MB 00:00 2024-08-14T06:50:07.941 INFO:teuthology.orchestra.run.smithi098.stdout:(1/160): ceph-19.0.0-5779.g488c4d2c.el9.x86_64. 37 kB/s | 6.6 kB 00:00 2024-08-14T06:50:07.941 INFO:teuthology.orchestra.run.smithi191.stdout:(73/160): python3-attrs-20.3.0-7.el9.noarch.rpm 1.3 MB/s | 87 kB 00:00 2024-08-14T06:50:07.984 INFO:teuthology.orchestra.run.smithi191.stdout:(74/160): python3-colorama-0.4.6-3.el9.noarch.r 1.4 MB/s | 61 kB 00:00 2024-08-14T06:50:08.067 INFO:teuthology.orchestra.run.smithi191.stdout:(75/160): postgresql-server-13.14-1.el9.x86_64. 21 MB/s | 5.8 MB 00:00 2024-08-14T06:50:08.093 INFO:teuthology.orchestra.run.smithi191.stdout:(76/160): python3-devel-3.9.18-3.el9.x86_64.rpm 2.2 MB/s | 244 kB 00:00 2024-08-14T06:50:08.118 INFO:teuthology.orchestra.run.smithi191.stdout:(77/160): python3-jinja2-2.11.3-6.el9.noarch.rp 4.8 MB/s | 248 kB 00:00 2024-08-14T06:50:08.141 INFO:teuthology.orchestra.run.smithi098.stdout:(2/160): ceph-fuse-19.0.0-5779.g488c4d2c.el9.x8 4.3 MB/s | 873 kB 00:00 2024-08-14T06:50:08.144 INFO:teuthology.orchestra.run.smithi191.stdout:(78/160): python3-jsonpatch-1.21-16.el9.noarch. 507 kB/s | 26 kB 00:00 2024-08-14T06:50:08.169 INFO:teuthology.orchestra.run.smithi191.stdout:(79/160): python3-jsonpointer-2.0-4.el9.noarch. 379 kB/s | 19 kB 00:00 2024-08-14T06:50:08.194 INFO:teuthology.orchestra.run.smithi191.stdout:(80/160): python3-mako-1.1.4-6.el9.noarch.rpm 3.3 MB/s | 172 kB 00:00 2024-08-14T06:50:08.216 INFO:teuthology.orchestra.run.smithi098.stdout:(3/160): ceph-immutable-object-cache-19.0.0-577 1.9 MB/s | 148 kB 00:00 2024-08-14T06:50:08.220 INFO:teuthology.orchestra.run.smithi191.stdout:(81/160): python3-markupsafe-1.1.1-12.el9.x86_6 686 kB/s | 35 kB 00:00 2024-08-14T06:50:08.300 INFO:teuthology.orchestra.run.smithi098.stdout:(4/160): ceph-base-19.0.0-5779.g488c4d2c.el9.x8 10 MB/s | 5.5 MB 00:00 2024-08-14T06:50:08.321 INFO:teuthology.orchestra.run.smithi191.stdout:(82/160): python3-babel-2.9.1-2.el9.noarch.rpm 15 MB/s | 6.0 MB 00:00 2024-08-14T06:50:08.346 INFO:teuthology.orchestra.run.smithi191.stdout:(83/160): python3-numpy-f2py-1.20.1-5.el9.x86_6 1.9 MB/s | 250 kB 00:00 2024-08-14T06:50:08.371 INFO:teuthology.orchestra.run.smithi191.stdout:(84/160): python3-oauthlib-3.1.1-5.el9.noarch.r 4.3 MB/s | 222 kB 00:00 2024-08-14T06:50:08.392 INFO:teuthology.orchestra.run.smithi098.stdout:(5/160): ceph-mgr-19.0.0-5779.g488c4d2c.el9.x86 17 MB/s | 1.5 MB 00:00 2024-08-14T06:50:08.397 INFO:teuthology.orchestra.run.smithi191.stdout:(85/160): python3-packaging-20.9-5.el9.noarch.r 1.5 MB/s | 77 kB 00:00 2024-08-14T06:50:08.422 INFO:teuthology.orchestra.run.smithi191.stdout:(86/160): python3-prettytable-0.7.2-27.el9.noar 839 kB/s | 42 kB 00:00 2024-08-14T06:50:08.450 INFO:teuthology.orchestra.run.smithi191.stdout:(87/160): python3-protobuf-3.14.0-13.el9.noarch 5.2 MB/s | 269 kB 00:00 2024-08-14T06:50:08.473 INFO:teuthology.orchestra.run.smithi191.stdout:(88/160): python3-pyasn1-0.4.8-6.el9.noarch.rpm 3.1 MB/s | 159 kB 00:00 2024-08-14T06:50:08.499 INFO:teuthology.orchestra.run.smithi191.stdout:(89/160): python3-pyasn1-modules-0.4.8-6.el9.no 5.4 MB/s | 279 kB 00:00 2024-08-14T06:50:08.524 INFO:teuthology.orchestra.run.smithi191.stdout:(90/160): python3-pytz-2021.1-5.el9.noarch.rpm 1.0 MB/s | 51 kB 00:00 2024-08-14T06:50:08.549 INFO:teuthology.orchestra.run.smithi191.stdout:(91/160): python3-requests-oauthlib-1.3.0-12.el 1.0 MB/s | 54 kB 00:00 2024-08-14T06:50:08.600 INFO:teuthology.orchestra.run.smithi098.stdout:(6/160): ceph-mds-19.0.0-5779.g488c4d2c.el9.x86 5.9 MB/s | 2.3 MB 00:00 2024-08-14T06:50:08.625 INFO:teuthology.orchestra.run.smithi191.stdout:(92/160): python3-numpy-1.20.1-5.el9.x86_64.rpm 12 MB/s | 5.1 MB 00:00 2024-08-14T06:50:08.650 INFO:teuthology.orchestra.run.smithi191.stdout:(93/160): python3-toml-0.10.2-6.el9.noarch.rpm 414 kB/s | 42 kB 00:00 2024-08-14T06:50:08.675 INFO:teuthology.orchestra.run.smithi191.stdout:(94/160): python3-wcwidth-0.2.5-8.el9.noarch.rp 880 kB/s | 44 kB 00:00 2024-08-14T06:50:08.684 INFO:teuthology.orchestra.run.smithi098.stdout:(7/160): ceph-mon-19.0.0-5779.g488c4d2c.el9.x86 17 MB/s | 4.9 MB 00:00 2024-08-14T06:50:08.701 INFO:teuthology.orchestra.run.smithi191.stdout:(95/160): qatlib-24.02.0-1.el9.x86_64.rpm 4.3 MB/s | 221 kB 00:00 2024-08-14T06:50:08.726 INFO:teuthology.orchestra.run.smithi191.stdout:(96/160): qatlib-service-24.02.0-1.el9.x86_64.r 712 kB/s | 36 kB 00:00 2024-08-14T06:50:08.752 INFO:teuthology.orchestra.run.smithi191.stdout:(97/160): qatzip-libs-1.2.0-1.el9.x86_64.rpm 940 kB/s | 47 kB 00:00 2024-08-14T06:50:08.777 INFO:teuthology.orchestra.run.smithi191.stdout:(98/160): socat-1.7.4.1-6.el9.x86_64.rpm 5.9 MB/s | 304 kB 00:00 2024-08-14T06:50:08.803 INFO:teuthology.orchestra.run.smithi191.stdout:(99/160): uuid-1.6.2-55.el9.x86_64.rpm 1.1 MB/s | 58 kB 00:00 2024-08-14T06:50:08.828 INFO:teuthology.orchestra.run.smithi191.stdout:(100/160): xfsprogs-devel-6.4.0-3.el9.x86_64.rp 1.9 MB/s | 96 kB 00:00 2024-08-14T06:50:08.854 INFO:teuthology.orchestra.run.smithi191.stdout:(101/160): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-08-14T06:50:08.879 INFO:teuthology.orchestra.run.smithi191.stdout:(102/160): zlib-devel-1.2.11-41.el9.x86_64.rpm 885 kB/s | 45 kB 00:00 2024-08-14T06:50:08.954 INFO:teuthology.orchestra.run.smithi191.stdout:(103/160): lua-devel-5.4.4-4.el9.x86_64.rpm 222 kB/s | 22 kB 00:00 2024-08-14T06:50:08.993 INFO:teuthology.orchestra.run.smithi098.stdout:(8/160): ceph-common-19.0.0-5779.g488c4d2c.el9. 16 MB/s | 20 MB 00:01 2024-08-14T06:50:09.035 INFO:teuthology.orchestra.run.smithi098.stdout:(9/160): ceph-selinux-19.0.0-5779.g488c4d2c.el9 607 kB/s | 25 kB 00:00 2024-08-14T06:50:09.147 INFO:teuthology.orchestra.run.smithi191.stdout:(104/160): python3-scipy-1.6.2-8.el9.x86_64.rpm 26 MB/s | 16 MB 00:00 2024-08-14T06:50:09.205 INFO:teuthology.orchestra.run.smithi191.stdout:(105/160): abseil-cpp-20211102.0-3.el9.x86_64.r 2.1 MB/s | 548 kB 00:00 2024-08-14T06:50:09.227 INFO:teuthology.orchestra.run.smithi098.stdout:(10/160): ceph-radosgw-19.0.0-5779.g488c4d2c.el 19 MB/s | 10 MB 00:00 2024-08-14T06:50:09.231 INFO:teuthology.orchestra.run.smithi191.stdout:(106/160): grpc-data-1.46.7-10.el9.noarch.rpm 863 kB/s | 19 kB 00:00 2024-08-14T06:50:09.256 INFO:teuthology.orchestra.run.smithi191.stdout:(107/160): protobuf-compiler-3.14.0-13.el9.x86_ 2.2 MB/s | 863 kB 00:00 2024-08-14T06:50:09.277 INFO:teuthology.orchestra.run.smithi098.stdout:(11/160): cephfs-mirror-19.0.0-5779.g488c4d2c.e 4.4 MB/s | 225 kB 00:00 2024-08-14T06:50:09.306 INFO:teuthology.orchestra.run.smithi191.stdout:(108/160): gperftools-libs-2.9.1-2.el9.x86_64.r 1.9 MB/s | 309 kB 00:00 2024-08-14T06:50:09.332 INFO:teuthology.orchestra.run.smithi191.stdout:(109/160): liboath-2.6.7-2.el9.x86_64.rpm 2.1 MB/s | 49 kB 00:00 2024-08-14T06:50:09.357 INFO:teuthology.orchestra.run.smithi191.stdout:(110/160): libarrow-doc-9.0.0-11.el9.noarch.rpm 257 kB/s | 26 kB 00:00 2024-08-14T06:50:09.361 INFO:teuthology.orchestra.run.smithi098.stdout:(12/160): cephfs-shell-19.0.0-5779.g488c4d2c.el 324 kB/s | 27 kB 00:00 2024-08-14T06:50:09.382 INFO:teuthology.orchestra.run.smithi191.stdout:(111/160): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-08-14T06:50:09.416 INFO:teuthology.orchestra.run.smithi191.stdout:(112/160): luarocks-3.9.2-5.el9.noarch.rpm 2.5 MB/s | 151 kB 00:00 2024-08-14T06:50:09.459 INFO:teuthology.orchestra.run.smithi191.stdout:(113/160): parquet-libs-9.0.0-11.el9.x86_64.rpm 11 MB/s | 839 kB 00:00 2024-08-14T06:50:09.461 INFO:teuthology.orchestra.run.smithi098.stdout:(13/160): libcephfs-devel-19.0.0-5779.g488c4d2c 317 kB/s | 32 kB 00:00 2024-08-14T06:50:09.484 INFO:teuthology.orchestra.run.smithi191.stdout:(114/160): python3-bcrypt-3.2.2-1.el9.x86_64.rp 1.7 MB/s | 43 kB 00:00 2024-08-14T06:50:09.509 INFO:teuthology.orchestra.run.smithi191.stdout:(115/160): python3-cachetools-4.2.4-1.el9.noarc 1.3 MB/s | 32 kB 00:00 2024-08-14T06:50:09.534 INFO:teuthology.orchestra.run.smithi191.stdout:(116/160): python3-certifi-2023.05.07-4.el9.noa 567 kB/s | 14 kB 00:00 2024-08-14T06:50:09.560 INFO:teuthology.orchestra.run.smithi191.stdout:(117/160): python3-asyncssh-2.13.2-5.el9.noarch 3.7 MB/s | 548 kB 00:00 2024-08-14T06:50:09.578 INFO:teuthology.orchestra.run.smithi098.stdout:(14/160): libcephfs2-19.0.0-5779.g488c4d2c.el9. 6.4 MB/s | 765 kB 00:00 2024-08-14T06:50:09.586 INFO:teuthology.orchestra.run.smithi191.stdout:(118/160): python3-cheroot-8.6.0-4.el9.noarch.r 3.3 MB/s | 172 kB 00:00 2024-08-14T06:50:09.619 INFO:teuthology.orchestra.run.smithi191.stdout:(119/160): python3-cherrypy-18.6.1-2.el9.noarch 5.9 MB/s | 358 kB 00:00 2024-08-14T06:50:09.645 INFO:teuthology.orchestra.run.smithi191.stdout:(120/160): python3-cmd2-2.4.2-2.el9.noarch.rpm 5.0 MB/s | 303 kB 00:00 2024-08-14T06:50:09.670 INFO:teuthology.orchestra.run.smithi098.stdout:(15/160): libcephsqlite-19.0.0-5779.g488c4d2c.e 1.7 MB/s | 158 kB 00:00 2024-08-14T06:50:09.670 INFO:teuthology.orchestra.run.smithi191.stdout:(121/160): python3-google-auth-2.32.0-1.el9.noa 4.2 MB/s | 216 kB 00:00 2024-08-14T06:50:09.704 INFO:teuthology.orchestra.run.smithi191.stdout:(122/160): python3-grpcio-tools-1.46.7-10.el9.x 4.2 MB/s | 144 kB 00:00 2024-08-14T06:50:09.729 INFO:teuthology.orchestra.run.smithi191.stdout:(123/160): python3-jaraco-8.2.1-3.el9.noarch.rp 426 kB/s | 11 kB 00:00 2024-08-14T06:50:09.755 INFO:teuthology.orchestra.run.smithi191.stdout:(124/160): python3-jaraco-classes-3.2.1-5.el9.n 706 kB/s | 18 kB 00:00 2024-08-14T06:50:09.762 INFO:teuthology.orchestra.run.smithi098.stdout:(16/160): librados-devel-19.0.0-5779.g488c4d2c. 1.3 MB/s | 127 kB 00:00 2024-08-14T06:50:09.780 INFO:teuthology.orchestra.run.smithi191.stdout:(125/160): python3-jaraco-collections-3.0.0-8.e 921 kB/s | 23 kB 00:00 2024-08-14T06:50:09.839 INFO:teuthology.orchestra.run.smithi191.stdout:(126/160): libarrow-9.0.0-11.el9.x86_64.rpm 7.3 MB/s | 4.4 MB 00:00 2024-08-14T06:50:09.856 INFO:teuthology.orchestra.run.smithi191.stdout:(127/160): python3-jaraco-functools-3.5.0-2.el9 258 kB/s | 19 kB 00:00 2024-08-14T06:50:09.862 INFO:teuthology.orchestra.run.smithi098.stdout:(17/160): libradosstriper1-19.0.0-5779.g488c4d2 4.9 MB/s | 504 kB 00:00 2024-08-14T06:50:09.873 INFO:teuthology.orchestra.run.smithi191.stdout:(128/160): python3-jaraco-text-3.2.0-6.el9.noar 583 kB/s | 20 kB 00:00 2024-08-14T06:50:09.899 INFO:teuthology.orchestra.run.smithi191.stdout:(129/160): python3-logutils-0.3.5-21.el9.noarch 1.8 MB/s | 46 kB 00:00 2024-08-14T06:50:09.924 INFO:teuthology.orchestra.run.smithi191.stdout:(130/160): python3-more-itertools-8.12.0-2.el9. 3.1 MB/s | 79 kB 00:00 2024-08-14T06:50:09.950 INFO:teuthology.orchestra.run.smithi191.stdout:(131/160): python3-natsort-7.1.1-5.el9.noarch.r 2.3 MB/s | 58 kB 00:00 2024-08-14T06:50:09.992 INFO:teuthology.orchestra.run.smithi191.stdout:(132/160): python3-grpcio-1.46.7-10.el9.x86_64. 5.9 MB/s | 2.0 MB 00:00 2024-08-14T06:50:10.017 INFO:teuthology.orchestra.run.smithi191.stdout:(133/160): python3-portend-3.1.0-2.el9.noarch.r 659 kB/s | 16 kB 00:00 2024-08-14T06:50:10.043 INFO:teuthology.orchestra.run.smithi191.stdout:(134/160): python3-pecan-1.4.2-3.el9.noarch.rpm 2.9 MB/s | 272 kB 00:00 2024-08-14T06:50:10.046 INFO:teuthology.orchestra.run.smithi098.stdout:(18/160): ceph-osd-19.0.0-5779.g488c4d2c.el9.x8 12 MB/s | 18 MB 00:01 2024-08-14T06:50:10.068 INFO:teuthology.orchestra.run.smithi191.stdout:(135/160): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.7 MB/s | 90 kB 00:00 2024-08-14T06:50:10.088 INFO:teuthology.orchestra.run.smithi098.stdout:(19/160): python3-ceph-argparse-19.0.0-5779.g48 1.1 MB/s | 45 kB 00:00 2024-08-14T06:50:10.093 INFO:teuthology.orchestra.run.smithi191.stdout:(136/160): python3-pyperclip-1.8.2-2.el9.noarch 562 kB/s | 28 kB 00:00 2024-08-14T06:50:10.119 INFO:teuthology.orchestra.run.smithi191.stdout:(137/160): python3-repoze-lru-0.7-16.el9.noarch 609 kB/s | 31 kB 00:00 2024-08-14T06:50:10.144 INFO:teuthology.orchestra.run.smithi191.stdout:(138/160): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2024-08-14T06:50:10.178 INFO:teuthology.orchestra.run.smithi191.stdout:(139/160): python3-kubernetes-26.1.0-2.el9.noar 3.2 MB/s | 1.0 MB 00:00 2024-08-14T06:50:10.180 INFO:teuthology.orchestra.run.smithi098.stdout:(20/160): librgw2-19.0.0-5779.g488c4d2c.el9.x86 17 MB/s | 5.3 MB 00:00 2024-08-14T06:50:10.203 INFO:teuthology.orchestra.run.smithi191.stdout:(140/160): python3-routes-2.5.1-5.el9.noarch.rp 1.7 MB/s | 188 kB 00:00 2024-08-14T06:50:10.205 INFO:teuthology.orchestra.run.smithi098.stdout:(21/160): python3-ceph-common-19.0.0-5779.g488c 1.2 MB/s | 138 kB 00:00 2024-08-14T06:50:10.229 INFO:teuthology.orchestra.run.smithi191.stdout:(141/160): python3-tempora-5.0.0-2.el9.noarch.r 425 kB/s | 36 kB 00:00 2024-08-14T06:50:10.230 INFO:teuthology.orchestra.run.smithi098.stdout:(22/160): python3-cephfs-19.0.0-5779.g488c4d2c. 3.7 MB/s | 188 kB 00:00 2024-08-14T06:50:10.254 INFO:teuthology.orchestra.run.smithi191.stdout:(142/160): python3-typing-extensions-4.12.2-2.e 981 kB/s | 74 kB 00:00 2024-08-14T06:50:10.256 INFO:teuthology.orchestra.run.smithi098.stdout:(23/160): python3-rados-19.0.0-5779.g488c4d2c.e 7.2 MB/s | 370 kB 00:00 2024-08-14T06:50:10.280 INFO:teuthology.orchestra.run.smithi191.stdout:(143/160): python3-websocket-client-1.2.3-2.el9 1.7 MB/s | 90 kB 00:00 2024-08-14T06:50:10.281 INFO:teuthology.orchestra.run.smithi098.stdout:(24/160): python3-rbd-19.0.0-5779.g488c4d2c.el9 6.9 MB/s | 356 kB 00:00 2024-08-14T06:50:10.305 INFO:teuthology.orchestra.run.smithi191.stdout:(144/160): python3-webob-1.8.7-6.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-08-14T06:50:10.306 INFO:teuthology.orchestra.run.smithi098.stdout:(25/160): python3-rgw-19.0.0-5779.g488c4d2c.el9 2.3 MB/s | 116 kB 00:00 2024-08-14T06:50:10.330 INFO:teuthology.orchestra.run.smithi191.stdout:(145/160): python3-zc-lockfile-2.0-10.el9.noarc 397 kB/s | 20 kB 00:00 2024-08-14T06:50:10.331 INFO:teuthology.orchestra.run.smithi098.stdout:(26/160): rbd-fuse-19.0.0-5779.g488c4d2c.el9.x8 1.7 MB/s | 86 kB 00:00 2024-08-14T06:50:10.356 INFO:teuthology.orchestra.run.smithi191.stdout:(146/160): re2-20211101-3.el9.x86_64.rpm 3.7 MB/s | 192 kB 00:00 2024-08-14T06:50:10.381 INFO:teuthology.orchestra.run.smithi191.stdout:(147/160): python3-werkzeug-2.0.3-3.el9.1.noarc 3.3 MB/s | 427 kB 00:00 2024-08-14T06:50:10.382 INFO:teuthology.orchestra.run.smithi098.stdout:(27/160): rbd-nbd-19.0.0-5779.g488c4d2c.el9.x86 3.3 MB/s | 170 kB 00:00 2024-08-14T06:50:10.423 INFO:teuthology.orchestra.run.smithi191.stdout:(148/160): thrift-0.15.0-2.el9.x86_64.rpm 17 MB/s | 1.6 MB 00:00 2024-08-14T06:50:10.457 INFO:teuthology.orchestra.run.smithi098.stdout:(28/160): rbd-mirror-19.0.0-5779.g488c4d2c.el9. 21 MB/s | 3.1 MB 00:00 2024-08-14T06:50:10.483 INFO:teuthology.orchestra.run.smithi098.stdout:(29/160): ceph-grafana-dashboards-19.0.0-5779.g 343 kB/s | 34 kB 00:00 2024-08-14T06:50:10.508 INFO:teuthology.orchestra.run.smithi098.stdout:(30/160): ceph-mgr-cephadm-19.0.0-5779.g488c4d2 2.9 MB/s | 150 kB 00:00 2024-08-14T06:50:10.523 INFO:teuthology.orchestra.run.smithi191.stdout:(149/160): NetworkManager-1.48.2-2.el9.x86_64.r 23 MB/s | 2.3 MB 00:00 2024-08-14T06:50:10.599 INFO:teuthology.orchestra.run.smithi191.stdout:(150/160): NetworkManager-libnm-1.48.2-2.el9.x8 24 MB/s | 1.8 MB 00:00 2024-08-14T06:50:10.641 INFO:teuthology.orchestra.run.smithi191.stdout:(151/160): NetworkManager-team-1.48.2-2.el9.x86 976 kB/s | 41 kB 00:00 2024-08-14T06:50:10.700 INFO:teuthology.orchestra.run.smithi191.stdout:(152/160): NetworkManager-tui-1.48.2-2.el9.x86_ 4.1 MB/s | 247 kB 00:00 2024-08-14T06:50:10.742 INFO:teuthology.orchestra.run.smithi191.stdout:(153/160): elfutils-debuginfod-client-0.191-4.e 878 kB/s | 37 kB 00:00 2024-08-14T06:50:10.792 INFO:teuthology.orchestra.run.smithi191.stdout:(154/160): elfutils-libelf-0.191-4.el9.x86_64.r 4.0 MB/s | 207 kB 00:00 2024-08-14T06:50:10.851 INFO:teuthology.orchestra.run.smithi191.stdout:(155/160): elfutils-libs-0.191-4.el9.x86_64.rpm 4.3 MB/s | 260 kB 00:00 2024-08-14T06:50:10.902 INFO:teuthology.orchestra.run.smithi098.stdout:(31/160): ceph-mgr-diskprediction-local-19.0.0- 19 MB/s | 7.4 MB 00:00 2024-08-14T06:50:10.918 INFO:teuthology.orchestra.run.smithi191.stdout:(156/160): openssl-3.2.2-2.el9.x86_64.rpm 20 MB/s | 1.4 MB 00:00 2024-08-14T06:50:10.959 INFO:teuthology.orchestra.run.smithi098.stdout:(32/160): ceph-mgr-modules-core-19.0.0-5779.g48 4.8 MB/s | 284 kB 00:00 2024-08-14T06:50:11.002 INFO:teuthology.orchestra.run.smithi191.stdout:(157/160): openssl-libs-3.2.2-2.el9.x86_64.rpm 29 MB/s | 2.4 MB 00:00 2024-08-14T06:50:11.060 INFO:teuthology.orchestra.run.smithi191.stdout:(158/160): xfsprogs-6.4.0-3.el9.x86_64.rpm 19 MB/s | 1.1 MB 00:00 2024-08-14T06:50:11.100 INFO:teuthology.orchestra.run.smithi098.stdout:(33/160): ceph-mgr-dashboard-19.0.0-5779.g488c4 23 MB/s | 14 MB 00:00 2024-08-14T06:50:11.126 INFO:teuthology.orchestra.run.smithi098.stdout:(34/160): ceph-mgr-rook-19.0.0-5779.g488c4d2c.e 296 kB/s | 49 kB 00:00 2024-08-14T06:50:11.151 INFO:teuthology.orchestra.run.smithi098.stdout:(35/160): ceph-prometheus-alerts-19.0.0-5779.g4 328 kB/s | 16 kB 00:00 2024-08-14T06:50:11.194 INFO:teuthology.orchestra.run.smithi191.stdout:(159/160): librados2-19.0.0-5779.g488c4d2c.el9. 4.0 MB/s | 3.4 MB 00:00 2024-08-14T06:50:11.569 INFO:teuthology.orchestra.run.smithi098.stdout:(36/160): ceph-test-19.0.0-5779.g488c4d2c.el9.x 17 MB/s | 44 MB 00:02 2024-08-14T06:50:11.593 INFO:teuthology.orchestra.run.smithi098.stdout:(37/160): ceph-volume-19.0.0-5779.g488c4d2c.el9 590 kB/s | 276 kB 00:00 2024-08-14T06:50:11.621 INFO:teuthology.orchestra.run.smithi098.stdout:(38/160): cephfs-top-19.0.0-5779.g488c4d2c.el9. 350 kB/s | 18 kB 00:00 2024-08-14T06:50:11.663 INFO:teuthology.orchestra.run.smithi098.stdout:(39/160): cephadm-19.0.0-5779.g488c4d2c.el9.noa 1.5 MB/s | 773 kB 00:00 2024-08-14T06:50:11.780 INFO:teuthology.orchestra.run.smithi098.stdout:(40/160): fuse-2.9.9-16.el9.x86_64.rpm 429 kB/s | 80 kB 00:00 2024-08-14T06:50:11.872 INFO:teuthology.orchestra.run.smithi098.stdout:(41/160): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.7 MB/s | 253 kB 00:00 2024-08-14T06:50:11.897 INFO:teuthology.orchestra.run.smithi098.stdout:(42/160): libquadmath-11.4.1-3.el9.x86_64.rpm 835 kB/s | 195 kB 00:00 2024-08-14T06:50:11.922 INFO:teuthology.orchestra.run.smithi098.stdout:(43/160): libgfortran-11.4.1-3.el9.x86_64.rpm 2.6 MB/s | 803 kB 00:00 2024-08-14T06:50:11.973 INFO:teuthology.orchestra.run.smithi098.stdout:(44/160): python3-ply-3.11-14.el9.noarch.rpm 1.4 MB/s | 106 kB 00:00 2024-08-14T06:50:11.998 INFO:teuthology.orchestra.run.smithi098.stdout:(45/160): python3-pycparser-2.20-6.el9.noarch.r 1.8 MB/s | 135 kB 00:00 2024-08-14T06:50:12.031 INFO:teuthology.orchestra.run.smithi098.stdout:(46/160): python3-cryptography-36.0.1-4.el9.x86 7.7 MB/s | 1.2 MB 00:00 2024-08-14T06:50:12.057 INFO:teuthology.orchestra.run.smithi098.stdout:(47/160): python3-requests-2.25.1-8.el9.noarch. 1.5 MB/s | 125 kB 00:00 2024-08-14T06:50:12.082 INFO:teuthology.orchestra.run.smithi098.stdout:(48/160): python3-urllib3-1.26.5-6.el9.noarch.r 2.5 MB/s | 215 kB 00:00 2024-08-14T06:50:12.566 INFO:teuthology.orchestra.run.smithi098.stdout:(49/160): boost-program-options-1.75.0-8.el9.x8 221 kB/s | 107 kB 00:00 2024-08-14T06:50:12.594 INFO:teuthology.orchestra.run.smithi191.stdout:(160/160): librbd1-19.0.0-5779.g488c4d2c.el9.x8 1.4 MB/s | 3.1 MB 00:02 2024-08-14T06:50:12.600 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-08-14T06:50:12.601 INFO:teuthology.orchestra.run.smithi191.stdout:Total 23 MB/s | 233 MB 00:10 2024-08-14T06:50:12.683 INFO:teuthology.orchestra.run.smithi098.stdout:(50/160): elfutils-libelf-devel-0.191-4.el9.x86 186 kB/s | 22 kB 00:00 2024-08-14T06:50:12.891 INFO:teuthology.orchestra.run.smithi098.stdout:(51/160): automake-1.16.2-8.el9.noarch.rpm 802 kB/s | 689 kB 00:00 2024-08-14T06:50:12.933 INFO:teuthology.orchestra.run.smithi098.stdout:(52/160): bison-3.7.4-5.el9.x86_64.rpm 1.1 MB/s | 943 kB 00:00 2024-08-14T06:50:13.034 INFO:teuthology.orchestra.run.smithi098.stdout:(53/160): flex-2.6.4-9.el9.x86_64.rpm 894 kB/s | 313 kB 00:00 2024-08-14T06:50:13.059 INFO:teuthology.orchestra.run.smithi098.stdout:(54/160): flexiblas-3.0.4-8.el9.x86_64.rpm 195 kB/s | 33 kB 00:00 2024-08-14T06:50:13.151 INFO:teuthology.orchestra.run.smithi098.stdout:(55/160): flexiblas-openblas-openmp-3.0.4-8.el9 156 kB/s | 18 kB 00:00 2024-08-14T06:50:13.176 INFO:teuthology.orchestra.run.smithi098.stdout:(56/160): libaio-devel-0.3.111-13.el9.x86_64.rp 95 kB/s | 11 kB 00:00 2024-08-14T06:50:13.268 INFO:teuthology.orchestra.run.smithi098.stdout:(57/160): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.3 MB/s | 160 kB 00:00 2024-08-14T06:50:13.293 INFO:teuthology.orchestra.run.smithi098.stdout:(58/160): librabbitmq-0.11.0-7.el9.x86_64.rpm 387 kB/s | 45 kB 00:00 2024-08-14T06:50:13.385 INFO:teuthology.orchestra.run.smithi098.stdout:(59/160): flexiblas-netlib-3.0.4-8.el9.x86_64.r 6.6 MB/s | 3.0 MB 00:00 2024-08-14T06:50:13.502 INFO:teuthology.orchestra.run.smithi098.stdout:(60/160): libuuid-devel-2.37.4-18.el9.x86_64.rp 218 kB/s | 25 kB 00:00 2024-08-14T06:50:13.569 INFO:teuthology.orchestra.run.smithi098.stdout:(61/160): librdkafka-1.6.1-102.el9.x86_64.rpm 2.2 MB/s | 662 kB 00:00 2024-08-14T06:50:13.620 INFO:teuthology.orchestra.run.smithi098.stdout:(62/160): libzstd-devel-1.5.1-2.el9.x86_64.rpm 405 kB/s | 47 kB 00:00 2024-08-14T06:50:13.695 INFO:teuthology.orchestra.run.smithi098.stdout:(63/160): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.3 MB/s | 292 kB 00:00 2024-08-14T06:50:13.728 INFO:teuthology.orchestra.run.smithi098.stdout:(64/160): libtool-2.4.6-46.el9.x86_64.rpm 1.3 MB/s | 581 kB 00:00 2024-08-14T06:50:13.754 INFO:teuthology.orchestra.run.smithi098.stdout:(65/160): lua-5.4.4-4.el9.x86_64.rpm 1.4 MB/s | 188 kB 00:00 2024-08-14T06:50:13.813 INFO:teuthology.orchestra.run.smithi098.stdout:(66/160): openblas-0.3.21-2.el9.x86_64.rpm 299 kB/s | 35 kB 00:00 2024-08-14T06:50:13.846 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-08-14T06:50:14.050 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-08-14T06:50:14.051 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-08-14T06:50:14.138 INFO:teuthology.orchestra.run.smithi098.stdout:(67/160): postgresql-13.14-1.el9.x86_64.rpm 4.9 MB/s | 1.6 MB 00:00 2024-08-14T06:50:14.272 INFO:teuthology.orchestra.run.smithi098.stdout:(68/160): postgresql-contrib-13.14-1.el9.x86_64 6.7 MB/s | 911 kB 00:00 2024-08-14T06:50:14.397 INFO:teuthology.orchestra.run.smithi098.stdout:(69/160): postgresql-private-libs-13.14-1.el9.x 1.0 MB/s | 133 kB 00:00 2024-08-14T06:50:16.247 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-08-14T06:50:16.247 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-08-14T06:50:16.323 INFO:teuthology.orchestra.run.smithi098.stdout:(70/160): openblas-openmp-0.3.21-2.el9.x86_64.r 1.8 MB/s | 4.7 MB 00:02 2024-08-14T06:50:16.823 INFO:teuthology.orchestra.run.smithi098.stdout:(71/160): protobuf-3.14.0-13.el9.x86_64.rpm 2.0 MB/s | 1.0 MB 00:00 2024-08-14T06:50:16.964 INFO:teuthology.orchestra.run.smithi098.stdout:(72/160): python3-attrs-20.3.0-7.el9.noarch.rpm 617 kB/s | 87 kB 00:00 2024-08-14T06:50:17.781 INFO:teuthology.orchestra.run.smithi098.stdout:(73/160): openssl-devel-3.2.2-2.el9.x86_64.rpm 1.1 MB/s | 4.4 MB 00:04 2024-08-14T06:50:17.898 INFO:teuthology.orchestra.run.smithi098.stdout:(74/160): python3-colorama-0.4.6-3.el9.noarch.r 525 kB/s | 61 kB 00:00 2024-08-14T06:50:18.290 INFO:teuthology.orchestra.run.smithi098.stdout:(75/160): python3-devel-3.9.18-3.el9.x86_64.rpm 623 kB/s | 244 kB 00:00 2024-08-14T06:50:18.699 INFO:teuthology.orchestra.run.smithi098.stdout:(76/160): python3-jinja2-2.11.3-6.el9.noarch.rp 608 kB/s | 248 kB 00:00 2024-08-14T06:50:18.816 INFO:teuthology.orchestra.run.smithi098.stdout:(77/160): python3-jsonpatch-1.21-16.el9.noarch. 219 kB/s | 26 kB 00:00 2024-08-14T06:50:18.933 INFO:teuthology.orchestra.run.smithi098.stdout:(78/160): python3-jsonpointer-2.0-4.el9.noarch. 164 kB/s | 19 kB 00:00 2024-08-14T06:50:19.008 INFO:teuthology.orchestra.run.smithi098.stdout:(79/160): python3-babel-2.9.1-2.el9.noarch.rpm 2.9 MB/s | 6.0 MB 00:02 2024-08-14T06:50:19.125 INFO:teuthology.orchestra.run.smithi098.stdout:(80/160): python3-markupsafe-1.1.1-12.el9.x86_6 297 kB/s | 35 kB 00:00 2024-08-14T06:50:19.242 INFO:teuthology.orchestra.run.smithi098.stdout:(81/160): python3-mako-1.1.4-6.el9.noarch.rpm 557 kB/s | 172 kB 00:00 2024-08-14T06:50:19.643 INFO:teuthology.orchestra.run.smithi098.stdout:(82/160): python3-numpy-f2py-1.20.1-5.el9.x86_6 626 kB/s | 250 kB 00:00 2024-08-14T06:50:20.043 INFO:teuthology.orchestra.run.smithi098.stdout:(83/160): python3-oauthlib-3.1.1-5.el9.noarch.r 555 kB/s | 222 kB 00:00 2024-08-14T06:50:20.251 INFO:teuthology.orchestra.run.smithi098.stdout:(84/160): python3-packaging-20.9-5.el9.noarch.r 371 kB/s | 77 kB 00:00 2024-08-14T06:50:20.385 INFO:teuthology.orchestra.run.smithi098.stdout:(85/160): python3-prettytable-0.7.2-27.el9.noar 316 kB/s | 42 kB 00:00 2024-08-14T06:50:20.452 INFO:teuthology.orchestra.run.smithi098.stdout:(86/160): python3-numpy-1.20.1-5.el9.x86_64.rpm 3.8 MB/s | 5.1 MB 00:01 2024-08-14T06:50:20.577 INFO:teuthology.orchestra.run.smithi098.stdout:(87/160): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.2 MB/s | 159 kB 00:00 2024-08-14T06:50:20.703 INFO:teuthology.orchestra.run.smithi098.stdout:(88/160): python3-pyasn1-modules-0.4.8-6.el9.no 2.2 MB/s | 279 kB 00:00 2024-08-14T06:50:20.795 INFO:teuthology.orchestra.run.smithi098.stdout:(89/160): postgresql-server-13.14-1.el9.x86_64. 925 kB/s | 5.8 MB 00:06 2024-08-14T06:50:20.820 INFO:teuthology.orchestra.run.smithi098.stdout:(90/160): python3-pytz-2021.1-5.el9.noarch.rpm 435 kB/s | 51 kB 00:00 2024-08-14T06:50:20.920 INFO:teuthology.orchestra.run.smithi098.stdout:(91/160): python3-protobuf-3.14.0-13.el9.noarch 503 kB/s | 269 kB 00:00 2024-08-14T06:50:20.937 INFO:teuthology.orchestra.run.smithi098.stdout:(92/160): python3-requests-oauthlib-1.3.0-12.el 376 kB/s | 54 kB 00:00 2024-08-14T06:50:21.029 INFO:teuthology.orchestra.run.smithi098.stdout:(93/160): python3-toml-0.10.2-6.el9.noarch.rpm 383 kB/s | 42 kB 00:00 2024-08-14T06:50:21.046 INFO:teuthology.orchestra.run.smithi098.stdout:(94/160): python3-wcwidth-0.2.5-8.el9.noarch.rp 406 kB/s | 44 kB 00:00 2024-08-14T06:50:21.155 INFO:teuthology.orchestra.run.smithi098.stdout:(95/160): qatlib-service-24.02.0-1.el9.x86_64.r 331 kB/s | 36 kB 00:00 2024-08-14T06:50:21.264 INFO:teuthology.orchestra.run.smithi098.stdout:(96/160): qatzip-libs-1.2.0-1.el9.x86_64.rpm 437 kB/s | 47 kB 00:00 2024-08-14T06:50:21.422 INFO:teuthology.orchestra.run.smithi098.stdout:(97/160): qatlib-24.02.0-1.el9.x86_64.rpm 562 kB/s | 221 kB 00:00 2024-08-14T06:50:21.565 INFO:teuthology.orchestra.run.smithi098.stdout:(98/160): uuid-1.6.2-55.el9.x86_64.rpm 407 kB/s | 58 kB 00:00 2024-08-14T06:50:21.773 INFO:teuthology.orchestra.run.smithi098.stdout:(99/160): socat-1.7.4.1-6.el9.x86_64.rpm 597 kB/s | 304 kB 00:00 2024-08-14T06:50:21.798 INFO:teuthology.orchestra.run.smithi098.stdout:(100/160): xfsprogs-devel-6.4.0-3.el9.x86_64.rp 409 kB/s | 96 kB 00:00 2024-08-14T06:50:21.924 INFO:teuthology.orchestra.run.smithi098.stdout:(101/160): zlib-devel-1.2.11-41.el9.x86_64.rpm 358 kB/s | 45 kB 00:00 2024-08-14T06:50:21.982 INFO:teuthology.orchestra.run.smithi098.stdout:(102/160): xmlstarlet-1.6.1-20.el9.x86_64.rpm 305 kB/s | 64 kB 00:00 2024-08-14T06:50:22.016 INFO:teuthology.orchestra.run.smithi098.stdout:(103/160): lua-devel-5.4.4-4.el9.x86_64.rpm 242 kB/s | 22 kB 00:00 2024-08-14T06:50:22.125 INFO:teuthology.orchestra.run.smithi098.stdout:(104/160): abseil-cpp-20211102.0-3.el9.x86_64.r 4.9 MB/s | 548 kB 00:00 2024-08-14T06:50:22.150 INFO:teuthology.orchestra.run.smithi098.stdout:(105/160): gperftools-libs-2.9.1-2.el9.x86_64.r 12 MB/s | 309 kB 00:00 2024-08-14T06:50:22.175 INFO:teuthology.orchestra.run.smithi098.stdout:(106/160): grpc-data-1.46.7-10.el9.noarch.rpm 782 kB/s | 19 kB 00:00 2024-08-14T06:50:22.226 INFO:teuthology.orchestra.run.smithi098.stdout:(107/160): protobuf-compiler-3.14.0-13.el9.x86_ 3.5 MB/s | 863 kB 00:00 2024-08-14T06:50:22.292 INFO:teuthology.orchestra.run.smithi098.stdout:(108/160): libarrow-doc-9.0.0-11.el9.noarch.rpm 389 kB/s | 26 kB 00:00 2024-08-14T06:50:22.326 INFO:teuthology.orchestra.run.smithi098.stdout:(109/160): liboath-2.6.7-2.el9.x86_64.rpm 1.4 MB/s | 49 kB 00:00 2024-08-14T06:50:22.385 INFO:teuthology.orchestra.run.smithi098.stdout:(110/160): libarrow-9.0.0-11.el9.x86_64.rpm 21 MB/s | 4.4 MB 00:00 2024-08-14T06:50:22.410 INFO:teuthology.orchestra.run.smithi098.stdout:(111/160): libunwind-1.6.2-1.el9.x86_64.rpm 805 kB/s | 67 kB 00:00 2024-08-14T06:50:22.435 INFO:teuthology.orchestra.run.smithi098.stdout:(112/160): luarocks-3.9.2-5.el9.noarch.rpm 2.9 MB/s | 151 kB 00:00 2024-08-14T06:50:22.478 INFO:teuthology.orchestra.run.smithi098.stdout:(113/160): python3-asyncssh-2.13.2-5.el9.noarch 13 MB/s | 548 kB 00:00 2024-08-14T06:50:22.503 INFO:teuthology.orchestra.run.smithi098.stdout:(114/160): python3-bcrypt-3.2.2-1.el9.x86_64.rp 1.7 MB/s | 43 kB 00:00 2024-08-14T06:50:22.528 INFO:teuthology.orchestra.run.smithi098.stdout:(115/160): python3-cachetools-4.2.4-1.el9.noarc 1.3 MB/s | 32 kB 00:00 2024-08-14T06:50:22.554 INFO:teuthology.orchestra.run.smithi098.stdout:(116/160): python3-certifi-2023.05.07-4.el9.noa 561 kB/s | 14 kB 00:00 2024-08-14T06:50:22.587 INFO:teuthology.orchestra.run.smithi098.stdout:(117/160): parquet-libs-9.0.0-11.el9.x86_64.rpm 4.6 MB/s | 839 kB 00:00 2024-08-14T06:50:22.613 INFO:teuthology.orchestra.run.smithi098.stdout:(118/160): python3-cheroot-8.6.0-4.el9.noarch.r 2.9 MB/s | 172 kB 00:00 2024-08-14T06:50:22.634 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-08-14T06:50:22.646 INFO:teuthology.orchestra.run.smithi098.stdout:(119/160): python3-cherrypy-18.6.1-2.el9.noarch 6.0 MB/s | 358 kB 00:00 2024-08-14T06:50:22.672 INFO:teuthology.orchestra.run.smithi098.stdout:(120/160): python3-cmd2-2.4.2-2.el9.noarch.rpm 5.1 MB/s | 303 kB 00:00 2024-08-14T06:50:22.697 INFO:teuthology.orchestra.run.smithi098.stdout:(121/160): python3-google-auth-2.32.0-1.el9.noa 4.2 MB/s | 216 kB 00:00 2024-08-14T06:50:22.730 INFO:teuthology.orchestra.run.smithi098.stdout:(122/160): python3-grpcio-tools-1.46.7-10.el9.x 4.2 MB/s | 144 kB 00:00 2024-08-14T06:50:22.756 INFO:teuthology.orchestra.run.smithi098.stdout:(123/160): python3-jaraco-8.2.1-3.el9.noarch.rp 426 kB/s | 11 kB 00:00 2024-08-14T06:50:22.758 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : openssl-libs-1:3.2.2-2.el9.x86_64 1/172 2024-08-14T06:50:22.781 INFO:teuthology.orchestra.run.smithi098.stdout:(124/160): python3-jaraco-classes-3.2.1-5.el9.n 708 kB/s | 18 kB 00:00 2024-08-14T06:50:22.802 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 2/172 2024-08-14T06:50:22.806 INFO:teuthology.orchestra.run.smithi098.stdout:(125/160): python3-jaraco-collections-3.0.0-8.e 931 kB/s | 23 kB 00:00 2024-08-14T06:50:22.832 INFO:teuthology.orchestra.run.smithi098.stdout:(126/160): python3-jaraco-functools-3.5.0-2.el9 777 kB/s | 19 kB 00:00 2024-08-14T06:50:22.857 INFO:teuthology.orchestra.run.smithi098.stdout:(127/160): python3-jaraco-text-3.2.0-6.el9.noar 783 kB/s | 20 kB 00:00 2024-08-14T06:50:22.870 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 3/172 2024-08-14T06:50:22.910 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/172 2024-08-14T06:50:22.916 INFO:teuthology.orchestra.run.smithi098.stdout:(128/160): python3-grpcio-1.46.7-10.el9.x86_64. 8.4 MB/s | 2.0 MB 00:00 2024-08-14T06:50:22.933 INFO:teuthology.orchestra.run.smithi098.stdout:(129/160): python3-logutils-0.3.5-21.el9.noarch 2.7 MB/s | 46 kB 00:00 2024-08-14T06:50:22.935 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 5/172 2024-08-14T06:50:22.958 INFO:teuthology.orchestra.run.smithi098.stdout:(130/160): python3-more-itertools-8.12.0-2.el9. 3.1 MB/s | 79 kB 00:00 2024-08-14T06:50:22.988 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 6/172 2024-08-14T06:50:22.991 INFO:teuthology.orchestra.run.smithi098.stdout:(131/160): python3-kubernetes-26.1.0-2.el9.noar 7.6 MB/s | 1.0 MB 00:00 2024-08-14T06:50:23.008 INFO:teuthology.orchestra.run.smithi098.stdout:(132/160): python3-natsort-7.1.1-5.el9.noarch.r 1.1 MB/s | 58 kB 00:00 2024-08-14T06:50:23.026 INFO:teuthology.orchestra.run.smithi098.stdout:(133/160): python3-portend-3.1.0-2.el9.noarch.r 977 kB/s | 16 kB 00:00 2024-08-14T06:50:23.051 INFO:teuthology.orchestra.run.smithi098.stdout:(134/160): python3-pyOpenSSL-21.0.0-1.el9.noarc 3.5 MB/s | 90 kB 00:00 2024-08-14T06:50:23.076 INFO:teuthology.orchestra.run.smithi098.stdout:(135/160): python3-pecan-1.4.2-3.el9.noarch.rpm 3.2 MB/s | 272 kB 00:00 2024-08-14T06:50:23.101 INFO:teuthology.orchestra.run.smithi098.stdout:(136/160): python3-pyperclip-1.8.2-2.el9.noarch 561 kB/s | 28 kB 00:00 2024-08-14T06:50:23.156 INFO:teuthology.orchestra.run.smithi098.stdout:(137/160): python3-repoze-lru-0.7-16.el9.noarch 387 kB/s | 31 kB 00:00 2024-08-14T06:50:23.186 INFO:teuthology.orchestra.run.smithi098.stdout:(138/160): python3-rsa-4.9-2.el9.noarch.rpm 2.0 MB/s | 59 kB 00:00 2024-08-14T06:50:23.219 INFO:teuthology.orchestra.run.smithi098.stdout:(139/160): python3-routes-2.5.1-5.el9.noarch.rp 1.6 MB/s | 188 kB 00:00 2024-08-14T06:50:23.244 INFO:teuthology.orchestra.run.smithi098.stdout:(140/160): python3-tempora-5.0.0-2.el9.noarch.r 610 kB/s | 36 kB 00:00 2024-08-14T06:50:23.270 INFO:teuthology.orchestra.run.smithi098.stdout:(141/160): python3-typing-extensions-4.12.2-2.e 1.4 MB/s | 74 kB 00:00 2024-08-14T06:50:23.287 INFO:teuthology.orchestra.run.smithi098.stdout:(142/160): python3-websocket-client-1.2.3-2.el9 5.3 MB/s | 90 kB 00:00 2024-08-14T06:50:23.312 INFO:teuthology.orchestra.run.smithi098.stdout:(143/160): python3-webob-1.8.7-6.el9.noarch.rpm 3.3 MB/s | 230 kB 00:00 2024-08-14T06:50:23.338 INFO:teuthology.orchestra.run.smithi098.stdout:(144/160): python3-zc-lockfile-2.0-10.el9.noarc 800 kB/s | 20 kB 00:00 2024-08-14T06:50:23.363 INFO:teuthology.orchestra.run.smithi098.stdout:(145/160): re2-20211101-3.el9.x86_64.rpm 7.5 MB/s | 192 kB 00:00 2024-08-14T06:50:23.388 INFO:teuthology.orchestra.run.smithi098.stdout:(146/160): python3-werkzeug-2.0.3-3.el9.1.noarc 4.1 MB/s | 427 kB 00:00 2024-08-14T06:50:23.439 INFO:teuthology.orchestra.run.smithi098.stdout:(147/160): thrift-0.15.0-2.el9.x86_64.rpm 21 MB/s | 1.6 MB 00:00 2024-08-14T06:50:23.455 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 7/172 2024-08-14T06:50:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : librados2-2:19.0.0-5779.g488c4d2c.el9.x86_64 8/172 2024-08-14T06:50:23.631 INFO:teuthology.orchestra.run.smithi098.stdout:(148/160): librados2-19.0.0-5779.g488c4d2c.el9. 14 MB/s | 3.4 MB 00:00 2024-08-14T06:50:23.645 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados2-2:19.0.0-5779.g488c4d2c.el9.x86_64 8/172 2024-08-14T06:50:23.672 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephfs2-2:19.0.0-5779.g488c4d2c.el9.x86_64 9/172 2024-08-14T06:50:23.714 INFO:teuthology.orchestra.run.smithi098.stdout:(149/160): librbd1-19.0.0-5779.g488c4d2c.el9.x8 11 MB/s | 3.1 MB 00:00 2024-08-14T06:50:23.759 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephfs2-2:19.0.0-5779.g488c4d2c.el9.x86_64 9/172 2024-08-14T06:50:23.786 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rados-2:19.0.0-5779.g488c4d2c.el9.x86_64 10/172 2024-08-14T06:50:23.833 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : postgresql-private-libs-13.14-1.el9.x86_64 11/172 2024-08-14T06:50:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : elfutils-libelf-0.191-4.el9.x86_64 12/172 2024-08-14T06:50:24.073 INFO:teuthology.orchestra.run.smithi098.stdout:(150/160): python3-scipy-1.6.2-8.el9.x86_64.rpm 5.0 MB/s | 16 MB 00:03 2024-08-14T06:50:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : postgresql-13.14-1.el9.x86_64 13/172 2024-08-14T06:50:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephsqlite-2:19.0.0-5779.g488c4d2c.el9.x86_64 14/172 2024-08-14T06:50:24.115 INFO:teuthology.orchestra.run.smithi098.stdout:(151/160): NetworkManager-libnm-1.48.2-2.el9.x8 4.5 MB/s | 1.8 MB 00:00 2024-08-14T06:50:24.173 INFO:teuthology.orchestra.run.smithi098.stdout:(152/160): NetworkManager-tui-1.48.2-2.el9.x86_ 4.1 MB/s | 247 kB 00:00 2024-08-14T06:50:24.203 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephsqlite-2:19.0.0-5779.g488c4d2c.el9.x86_64 14/172 2024-08-14T06:50:24.224 INFO:teuthology.orchestra.run.smithi098.stdout:(153/160): elfutils-debuginfod-client-0.191-4.e 735 kB/s | 37 kB 00:00 2024-08-14T06:50:24.232 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libradosstriper1-2:19.0.0-5779.g488c4d2c.el9.x86 15/172 2024-08-14T06:50:24.282 INFO:teuthology.orchestra.run.smithi098.stdout:(154/160): elfutils-libelf-0.191-4.el9.x86_64.r 3.5 MB/s | 207 kB 00:00 2024-08-14T06:50:24.294 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libradosstriper1-2:19.0.0-5779.g488c4d2c.el9.x86 15/172 2024-08-14T06:50:24.316 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 16/172 2024-08-14T06:50:24.336 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 16/172 2024-08-14T06:50:24.341 INFO:teuthology.orchestra.run.smithi098.stdout:(155/160): elfutils-libs-0.191-4.el9.x86_64.rpm 4.3 MB/s | 260 kB 00:00 2024-08-14T06:50:24.416 INFO:teuthology.orchestra.run.smithi098.stdout:(156/160): openssl-3.2.2-2.el9.x86_64.rpm 18 MB/s | 1.4 MB 00:00 2024-08-14T06:50:24.478 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 17/172 2024-08-14T06:50:24.525 INFO:teuthology.orchestra.run.smithi098.stdout:(157/160): openssl-libs-3.2.2-2.el9.x86_64.rpm 22 MB/s | 2.4 MB 00:00 2024-08-14T06:50:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 17/172 2024-08-14T06:50:24.600 INFO:teuthology.orchestra.run.smithi098.stdout:(158/160): xfsprogs-6.4.0-3.el9.x86_64.rpm 14 MB/s | 1.1 MB 00:00 2024-08-14T06:50:24.848 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : xfsprogs-6.4.0-3.el9.x86_64 18/172 2024-08-14T06:50:25.004 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : NetworkManager-libnm-1:1.48.2-2.el9.x86_64 19/172 2024-08-14T06:50:25.063 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: NetworkManager-libnm-1:1.48.2-2.el9.x86_64 19/172 2024-08-14T06:50:25.176 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: NetworkManager-1:1.48.2-2.el9.x86_64 20/172 2024-08-14T06:50:25.249 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : NetworkManager-1:1.48.2-2.el9.x86_64 20/172 2024-08-14T06:50:25.449 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: NetworkManager-1:1.48.2-2.el9.x86_64 20/172 2024-08-14T06:50:25.555 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : re2-1:20211101-3.el9.x86_64 21/172 2024-08-14T06:50:25.587 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 22/172 2024-08-14T06:50:25.592 INFO:teuthology.orchestra.run.smithi098.stdout:(159/160): NetworkManager-team-1.48.2-2.el9.x86 27 kB/s | 41 kB 00:01 2024-08-14T06:50:25.630 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : liboath-2.6.7-2.el9.x86_64 23/172 2024-08-14T06:50:25.696 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 24/172 2024-08-14T06:50:25.741 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 25/172 2024-08-14T06:50:25.773 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-packaging-20.9-5.el9.noarch 26/172 2024-08-14T06:50:25.817 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 27/172 2024-08-14T06:50:25.849 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 28/172 2024-08-14T06:50:25.878 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : lua-5.4.4-4.el9.x86_64 29/172 2024-08-14T06:50:25.947 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 30/172 2024-08-14T06:50:26.009 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 31/172 2024-08-14T06:50:26.039 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 32/172 2024-08-14T06:50:26.075 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 33/172 2024-08-14T06:50:26.152 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 34/172 2024-08-14T06:50:26.186 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-ceph-common-2:19.0.0-5779.g488c4d2c.el9. 35/172 2024-08-14T06:50:26.236 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-ceph-argparse-2:19.0.0-5779.g488c4d2c.el 36/172 2024-08-14T06:50:26.259 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cephfs-2:19.0.0-5779.g488c4d2c.el9.x86_6 37/172 2024-08-14T06:50:26.322 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 38/172 2024-08-14T06:50:26.382 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : luarocks-3.9.2-5.el9.noarch 39/172 2024-08-14T06:50:26.458 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 40/172 2024-08-14T06:50:26.594 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 41/172 2024-08-14T06:50:26.645 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 42/172 2024-08-14T06:50:27.188 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rsa-4.9-2.el9.noarch 43/172 2024-08-14T06:50:27.275 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 44/172 2024-08-14T06:50:27.312 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 45/172 2024-08-14T06:50:27.349 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 46/172 2024-08-14T06:50:27.390 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : elfutils-libs-0.191-4.el9.x86_64 47/172 2024-08-14T06:50:27.428 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librados-devel-2:19.0.0-5779.g488c4d2c.el9.x86_6 48/172 2024-08-14T06:50:27.471 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 49/172 2024-08-14T06:50:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 50/172 2024-08-14T06:50:27.699 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : openssl-1:3.2.2-2.el9.x86_64 51/172 2024-08-14T06:50:27.792 INFO:teuthology.orchestra.run.smithi098.stdout:(160/160): NetworkManager-1.48.2-2.el9.x86_64.r 568 kB/s | 2.3 MB 00:04 2024-08-14T06:50:27.792 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 52/172 2024-08-14T06:50:27.796 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2024-08-14T06:50:27.796 INFO:teuthology.orchestra.run.smithi098.stdout:Total 11 MB/s | 233 MB 00:20 2024-08-14T06:50:27.856 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 53/172 2024-08-14T06:50:27.929 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 54/172 2024-08-14T06:50:27.961 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 55/172 2024-08-14T06:50:27.997 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 56/172 2024-08-14T06:50:28.044 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 57/172 2024-08-14T06:50:28.088 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 58/172 2024-08-14T06:50:28.132 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 59/172 2024-08-14T06:50:28.245 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 60/172 2024-08-14T06:50:28.554 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 61/172 2024-08-14T06:50:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 62/172 2024-08-14T06:50:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 63/172 2024-08-14T06:50:28.750 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 64/172 2024-08-14T06:50:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-google-auth-1:2.32.0-1.el9.noarch 65/172 2024-08-14T06:50:28.835 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 66/172 2024-08-14T06:50:28.869 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 67/172 2024-08-14T06:50:28.894 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 68/172 2024-08-14T06:50:28.985 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 69/172 2024-08-14T06:50:29.002 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2024-08-14T06:50:29.122 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 70/172 2024-08-14T06:50:29.153 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 71/172 2024-08-14T06:50:29.181 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : zlib-devel-1.2.11-41.el9.x86_64 72/172 2024-08-14T06:50:29.204 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2024-08-14T06:50:29.205 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2024-08-14T06:50:29.216 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 73/172 2024-08-14T06:50:29.255 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : uuid-1.6.2-55.el9.x86_64 74/172 2024-08-14T06:50:29.287 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 75/172 2024-08-14T06:50:29.317 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 76/172 2024-08-14T06:50:29.352 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 77/172 2024-08-14T06:50:29.386 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 78/172 2024-08-14T06:50:29.427 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 79/172 2024-08-14T06:50:29.462 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 80/172 2024-08-14T06:50:29.549 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 81/172 2024-08-14T06:50:29.604 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 82/172 2024-08-14T06:50:29.641 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 83/172 2024-08-14T06:50:29.744 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 84/172 2024-08-14T06:50:29.793 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 85/172 2024-08-14T06:50:30.520 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 86/172 2024-08-14T06:50:30.575 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 87/172 2024-08-14T06:50:30.609 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 88/172 2024-08-14T06:50:30.713 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 89/172 2024-08-14T06:50:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 90/172 2024-08-14T06:50:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-colorama-0.4.6-3.el9.noarch 91/172 2024-08-14T06:50:30.916 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 92/172 2024-08-14T06:50:30.947 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 93/172 2024-08-14T06:50:31.124 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : openblas-0.3.21-2.el9.x86_64 94/172 2024-08-14T06:50:31.153 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 95/172 2024-08-14T06:50:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 96/172 2024-08-14T06:50:31.380 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2024-08-14T06:50:31.381 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2024-08-14T06:50:31.883 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 97/172 2024-08-14T06:50:32.005 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 98/172 2024-08-14T06:50:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 99/172 2024-08-14T06:50:33.348 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 100/172 2024-08-14T06:50:33.376 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 101/172 2024-08-14T06:50:33.410 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 102/172 2024-08-14T06:50:33.740 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 103/172 2024-08-14T06:50:33.782 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : librbd1-2:19.0.0-5779.g488c4d2c.el9.x86_64 104/172 2024-08-14T06:50:33.995 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librbd1-2:19.0.0-5779.g488c4d2c.el9.x86_64 104/172 2024-08-14T06:50:34.021 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 105/172 2024-08-14T06:50:34.066 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 106/172 2024-08-14T06:50:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 107/172 2024-08-14T06:50:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librgw2-2:19.0.0-5779.g488c4d2c.el9.x86_64 108/172 2024-08-14T06:50:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librgw2-2:19.0.0-5779.g488c4d2c.el9.x86_64 108/172 2024-08-14T06:50:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 109/172 2024-08-14T06:50:37.452 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 110/172 2024-08-14T06:50:37.487 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 110/172 2024-08-14T06:50:37.871 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2024-08-14T06:50:37.963 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : openssl-libs-1:3.2.2-2.el9.x86_64 1/172 2024-08-14T06:50:38.002 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 2/172 2024-08-14T06:50:38.064 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 110/172 2024-08-14T06:50:38.065 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 3/172 2024-08-14T06:50:38.097 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/172 2024-08-14T06:50:38.100 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-base-2:19.0.0-5779.g488c4d2c.el9.x86_64 111/172 2024-08-14T06:50:38.123 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 5/172 2024-08-14T06:50:38.179 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 6/172 2024-08-14T06:50:38.199 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-base-2:19.0.0-5779.g488c4d2c.el9.x86_64 111/172 2024-08-14T06:50:38.199 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-14T06:50:38.200 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-14T06:50:38.200 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:38.227 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-selinux-2:19.0.0-5779.g488c4d2c.el9.x86_64 112/172 2024-08-14T06:50:38.529 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 7/172 2024-08-14T06:50:38.649 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : librados2-2:19.0.0-5779.g488c4d2c.el9.x86_64 8/172 2024-08-14T06:50:38.776 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librados2-2:19.0.0-5779.g488c4d2c.el9.x86_64 8/172 2024-08-14T06:50:38.800 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libcephfs2-2:19.0.0-5779.g488c4d2c.el9.x86_64 9/172 2024-08-14T06:50:38.895 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libcephfs2-2:19.0.0-5779.g488c4d2c.el9.x86_64 9/172 2024-08-14T06:50:38.922 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-rados-2:19.0.0-5779.g488c4d2c.el9.x86_64 10/172 2024-08-14T06:50:38.960 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : postgresql-private-libs-13.14-1.el9.x86_64 11/172 2024-08-14T06:50:39.159 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : elfutils-libelf-0.191-4.el9.x86_64 12/172 2024-08-14T06:50:39.220 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : postgresql-13.14-1.el9.x86_64 13/172 2024-08-14T06:50:39.248 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libcephsqlite-2:19.0.0-5779.g488c4d2c.el9.x86_64 14/172 2024-08-14T06:50:39.337 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libcephsqlite-2:19.0.0-5779.g488c4d2c.el9.x86_64 14/172 2024-08-14T06:50:39.366 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libradosstriper1-2:19.0.0-5779.g488c4d2c.el9.x86 15/172 2024-08-14T06:50:39.411 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libradosstriper1-2:19.0.0-5779.g488c4d2c.el9.x86 15/172 2024-08-14T06:50:39.433 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 16/172 2024-08-14T06:50:39.457 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 16/172 2024-08-14T06:50:39.660 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 17/172 2024-08-14T06:50:39.751 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 17/172 2024-08-14T06:50:40.020 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : xfsprogs-6.4.0-3.el9.x86_64 18/172 2024-08-14T06:50:40.159 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : NetworkManager-libnm-1:1.48.2-2.el9.x86_64 19/172 2024-08-14T06:50:40.219 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: NetworkManager-libnm-1:1.48.2-2.el9.x86_64 19/172 2024-08-14T06:50:40.319 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: NetworkManager-1:1.48.2-2.el9.x86_64 20/172 2024-08-14T06:50:40.396 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : NetworkManager-1:1.48.2-2.el9.x86_64 20/172 2024-08-14T06:50:40.617 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: NetworkManager-1:1.48.2-2.el9.x86_64 20/172 2024-08-14T06:50:40.725 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : re2-1:20211101-3.el9.x86_64 21/172 2024-08-14T06:50:40.758 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 22/172 2024-08-14T06:50:40.801 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : liboath-2.6.7-2.el9.x86_64 23/172 2024-08-14T06:50:40.866 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 24/172 2024-08-14T06:50:40.911 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 25/172 2024-08-14T06:50:40.944 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-packaging-20.9-5.el9.noarch 26/172 2024-08-14T06:50:40.987 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 27/172 2024-08-14T06:50:41.020 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 28/172 2024-08-14T06:50:41.073 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : lua-5.4.4-4.el9.x86_64 29/172 2024-08-14T06:50:41.141 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 30/172 2024-08-14T06:50:41.195 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 31/172 2024-08-14T06:50:41.234 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 32/172 2024-08-14T06:50:41.269 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 33/172 2024-08-14T06:50:41.346 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 34/172 2024-08-14T06:50:41.380 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-ceph-common-2:19.0.0-5779.g488c4d2c.el9. 35/172 2024-08-14T06:50:41.428 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-ceph-argparse-2:19.0.0-5779.g488c4d2c.el 36/172 2024-08-14T06:50:41.460 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cephfs-2:19.0.0-5779.g488c4d2c.el9.x86_6 37/172 2024-08-14T06:50:41.532 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 38/172 2024-08-14T06:50:41.583 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : luarocks-3.9.2-5.el9.noarch 39/172 2024-08-14T06:50:41.659 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 40/172 2024-08-14T06:50:41.793 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 41/172 2024-08-14T06:50:41.846 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 42/172 2024-08-14T06:50:42.375 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-rsa-4.9-2.el9.noarch 43/172 2024-08-14T06:50:42.449 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 44/172 2024-08-14T06:50:42.486 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 45/172 2024-08-14T06:50:42.527 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 46/172 2024-08-14T06:50:42.572 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : elfutils-libs-0.191-4.el9.x86_64 47/172 2024-08-14T06:50:42.610 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : librados-devel-2:19.0.0-5779.g488c4d2c.el9.x86_6 48/172 2024-08-14T06:50:42.653 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 49/172 2024-08-14T06:50:42.802 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 50/172 2024-08-14T06:50:42.863 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : openssl-1:3.2.2-2.el9.x86_64 51/172 2024-08-14T06:50:42.955 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 52/172 2024-08-14T06:50:43.029 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 53/172 2024-08-14T06:50:43.101 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 54/172 2024-08-14T06:50:43.132 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 55/172 2024-08-14T06:50:43.168 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 56/172 2024-08-14T06:50:43.215 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 57/172 2024-08-14T06:50:43.250 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 58/172 2024-08-14T06:50:43.294 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 59/172 2024-08-14T06:50:43.337 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 60/172 2024-08-14T06:50:43.471 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 61/172 2024-08-14T06:50:43.513 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 62/172 2024-08-14T06:50:43.554 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 63/172 2024-08-14T06:50:43.653 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 64/172 2024-08-14T06:50:43.688 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-google-auth-1:2.32.0-1.el9.noarch 65/172 2024-08-14T06:50:43.729 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 66/172 2024-08-14T06:50:43.762 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 67/172 2024-08-14T06:50:43.786 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 68/172 2024-08-14T06:50:43.879 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 69/172 2024-08-14T06:50:44.013 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 70/172 2024-08-14T06:50:44.053 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 71/172 2024-08-14T06:50:44.080 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : zlib-devel-1.2.11-41.el9.x86_64 72/172 2024-08-14T06:50:44.118 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 73/172 2024-08-14T06:50:44.154 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : uuid-1.6.2-55.el9.x86_64 74/172 2024-08-14T06:50:44.186 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 75/172 2024-08-14T06:50:44.216 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 76/172 2024-08-14T06:50:44.250 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 77/172 2024-08-14T06:50:44.284 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 78/172 2024-08-14T06:50:44.325 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 79/172 2024-08-14T06:50:44.360 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 80/172 2024-08-14T06:50:44.447 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 81/172 2024-08-14T06:50:44.497 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 82/172 2024-08-14T06:50:44.538 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 83/172 2024-08-14T06:50:44.641 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 84/172 2024-08-14T06:50:44.698 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 85/172 2024-08-14T06:50:45.422 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 86/172 2024-08-14T06:50:45.470 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 87/172 2024-08-14T06:50:45.504 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 88/172 2024-08-14T06:50:45.605 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 89/172 2024-08-14T06:50:45.651 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 90/172 2024-08-14T06:50:45.696 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-colorama-0.4.6-3.el9.noarch 91/172 2024-08-14T06:50:45.784 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 92/172 2024-08-14T06:50:45.840 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 93/172 2024-08-14T06:50:46.017 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : openblas-0.3.21-2.el9.x86_64 94/172 2024-08-14T06:50:46.046 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 95/172 2024-08-14T06:50:46.124 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 96/172 2024-08-14T06:50:46.784 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 97/172 2024-08-14T06:50:46.904 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 98/172 2024-08-14T06:50:48.152 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 99/172 2024-08-14T06:50:48.276 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 100/172 2024-08-14T06:50:48.304 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 101/172 2024-08-14T06:50:48.338 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 102/172 2024-08-14T06:50:48.666 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 103/172 2024-08-14T06:50:48.701 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : librbd1-2:19.0.0-5779.g488c4d2c.el9.x86_64 104/172 2024-08-14T06:50:48.796 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librbd1-2:19.0.0-5779.g488c4d2c.el9.x86_64 104/172 2024-08-14T06:50:48.822 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-rbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 105/172 2024-08-14T06:50:48.862 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 106/172 2024-08-14T06:50:49.357 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 107/172 2024-08-14T06:50:49.395 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : librgw2-2:19.0.0-5779.g488c4d2c.el9.x86_64 108/172 2024-08-14T06:50:49.467 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librgw2-2:19.0.0-5779.g488c4d2c.el9.x86_64 108/172 2024-08-14T06:50:49.497 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-rgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 109/172 2024-08-14T06:50:49.910 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-selinux-2:19.0.0-5779.g488c4d2c.el9.x86_64 112/172 2024-08-14T06:50:49.910 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /sys 2024-08-14T06:50:49.911 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /proc 2024-08-14T06:50:49.911 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /mnt 2024-08-14T06:50:49.911 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /var/tmp 2024-08-14T06:50:49.911 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /home 2024-08-14T06:50:49.911 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /root 2024-08-14T06:50:49.911 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /tmp 2024-08-14T06:50:49.911 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:50.094 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-volume-2:19.0.0-5779.g488c4d2c.el9.noarch 113/172 2024-08-14T06:50:50.110 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-volume-2:19.0.0-5779.g488c4d2c.el9.noarch 113/172 2024-08-14T06:50:50.110 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:50:50.110 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-08-14T06:50:50.110 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:51.688 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-osd-2:19.0.0-5779.g488c4d2c.el9.x86_64 114/172 2024-08-14T06:50:51.715 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-osd-2:19.0.0-5779.g488c4d2c.el9.x86_64 114/172 2024-08-14T06:50:51.715 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:50:51.715 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-14T06:50:51.715 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-14T06:50:51.715 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-14T06:50:51.715 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:51.821 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 110/172 2024-08-14T06:50:51.853 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 110/172 2024-08-14T06:50:51.939 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mds-2:19.0.0-5779.g488c4d2c.el9.x86_64 115/172 2024-08-14T06:50:51.962 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mds-2:19.0.0-5779.g488c4d2c.el9.x86_64 115/172 2024-08-14T06:50:51.962 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:50:51.962 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-14T06:50:51.962 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-14T06:50:51.963 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-14T06:50:51.963 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:52.418 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 110/172 2024-08-14T06:50:52.423 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mon-2:19.0.0-5779.g488c4d2c.el9.x86_64 116/172 2024-08-14T06:50:52.446 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mon-2:19.0.0-5779.g488c4d2c.el9.x86_64 116/172 2024-08-14T06:50:52.447 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:50:52.447 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-14T06:50:52.447 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-14T06:50:52.447 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-14T06:50:52.447 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:52.553 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : automake-1.16.2-8.el9.noarch 117/172 2024-08-14T06:50:52.610 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-ply-3.11-14.el9.noarch 118/172 2024-08-14T06:50:52.657 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-base-2:19.0.0-5779.g488c4d2c.el9.x86_64 111/172 2024-08-14T06:50:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 119/172 2024-08-14T06:50:52.753 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-base-2:19.0.0-5779.g488c4d2c.el9.x86_64 111/172 2024-08-14T06:50:52.753 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-14T06:50:52.753 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-14T06:50:52.753 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:50:52.790 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-selinux-2:19.0.0-5779.g488c4d2c.el9.x86_64 112/172 2024-08-14T06:50:52.853 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 120/172 2024-08-14T06:50:52.923 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 121/172 2024-08-14T06:50:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 122/172 2024-08-14T06:50:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 123/172 2024-08-14T06:50:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 124/172 2024-08-14T06:50:53.265 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 125/172 2024-08-14T06:50:53.297 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 126/172 2024-08-14T06:50:53.325 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : fuse-2.9.9-16.el9.x86_64 127/172 2024-08-14T06:50:53.737 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: cephadm-2:19.0.0-5779.g488c4d2c.el9.noarch 128/172 2024-08-14T06:50:53.765 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : cephadm-2:19.0.0-5779.g488c4d2c.el9.noarch 128/172 2024-08-14T06:50:53.797 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-5779.g488c4d2c.e 129/172 2024-08-14T06:50:53.862 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-5779.g488c4d2c. 130/172 2024-08-14T06:50:53.888 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-5779.g488c4d2c.el9.noa 131/172 2024-08-14T06:50:57.091 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-5779.g488c4d2c.el9.noa 131/172 2024-08-14T06:50:57.144 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-5779.g488c4d2c.el9.n 132/172 2024-08-14T06:50:58.109 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-5779.g488c4d2c.el9.n 132/172 2024-08-14T06:50:58.136 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-5779.g488 133/172 2024-08-14T06:50:58.251 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-5779.g488 133/172 2024-08-14T06:50:58.411 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-5779.g488c4d2c.el 134/172 2024-08-14T06:50:58.438 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-2:19.0.0-5779.g488c4d2c.el9.x86_64 135/172 2024-08-14T06:50:58.462 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-2:19.0.0-5779.g488c4d2c.el9.x86_64 135/172 2024-08-14T06:50:58.462 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:50:58.462 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-14T06:50:58.462 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-14T06:50:58.462 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-14T06:50:58.462 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:58.522 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-rook-2:19.0.0-5779.g488c4d2c.el9.noarch 136/172 2024-08-14T06:50:58.534 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-5779.g488c4d2c.el9.noarch 136/172 2024-08-14T06:50:58.632 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-2:19.0.0-5779.g488c4d2c.el9.x86_64 137/172 2024-08-14T06:50:58.690 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 138/172 2024-08-14T06:50:59.645 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libtool-2.4.6-46.el9.x86_64 139/172 2024-08-14T06:50:59.682 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-radosgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 140/172 2024-08-14T06:50:59.706 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 140/172 2024-08-14T06:50:59.706 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:50:59.707 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-14T06:50:59.707 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-14T06:50:59.707 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-14T06:50:59.707 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:59.750 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-5779.g488c4 141/172 2024-08-14T06:50:59.772 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-5779.g488c4 141/172 2024-08-14T06:50:59.773 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:50:59.773 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-14T06:50:59.773 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:50:59.825 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : cephfs-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 142/172 2024-08-14T06:50:59.849 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 142/172 2024-08-14T06:50:59.849 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:50:59.849 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-08-14T06:50:59.849 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-08-14T06:50:59.849 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-08-14T06:50:59.849 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:00.176 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 143/172 2024-08-14T06:51:00.199 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: rbd-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 143/172 2024-08-14T06:51:00.200 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:51:00.200 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-14T06:51:00.200 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-14T06:51:00.200 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-14T06:51:00.200 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:04.571 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-test-2:19.0.0-5779.g488c4d2c.el9.x86_64 144/172 2024-08-14T06:51:04.605 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-selinux-2:19.0.0-5779.g488c4d2c.el9.x86_64 112/172 2024-08-14T06:51:04.605 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /sys 2024-08-14T06:51:04.605 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /proc 2024-08-14T06:51:04.605 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /mnt 2024-08-14T06:51:04.605 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /var/tmp 2024-08-14T06:51:04.605 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /home 2024-08-14T06:51:04.605 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /root 2024-08-14T06:51:04.605 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /tmp 2024-08-14T06:51:04.605 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:04.612 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 145/172 2024-08-14T06:51:04.653 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-nbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 146/172 2024-08-14T06:51:04.685 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : xfsprogs-devel-6.4.0-3.el9.x86_64 147/172 2024-08-14T06:51:04.712 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : elfutils-libelf-devel-0.191-4.el9.x86_64 148/172 2024-08-14T06:51:04.773 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-volume-2:19.0.0-5779.g488c4d2c.el9.noarch 113/172 2024-08-14T06:51:04.788 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-volume-2:19.0.0-5779.g488c4d2c.el9.noarch 113/172 2024-08-14T06:51:04.788 INFO:teuthology.orchestra.run.smithi098.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:51:04.788 INFO:teuthology.orchestra.run.smithi098.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-08-14T06:51:04.788 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:04.904 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : cephfs-shell-2:19.0.0-5779.g488c4d2c.el9.x86_64 149/172 2024-08-14T06:51:04.950 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : postgresql-contrib-13.14-1.el9.x86_64 150/172 2024-08-14T06:51:04.983 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephfs-devel-2:19.0.0-5779.g488c4d2c.el9.x86_ 151/172 2024-08-14T06:51:05.015 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : elfutils-debuginfod-client-0.191-4.el9.x86_64 152/172 2024-08-14T06:51:05.044 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : NetworkManager-team-1:1.48.2-2.el9.x86_64 153/172 2024-08-14T06:51:05.091 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : NetworkManager-tui-1:1.48.2-2.el9.x86_64 154/172 2024-08-14T06:51:05.702 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 155/172 2024-08-14T06:51:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : postgresql-server-13.14-1.el9.x86_64 155/172 2024-08-14T06:51:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 155/172 2024-08-14T06:51:06.368 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-osd-2:19.0.0-5779.g488c4d2c.el9.x86_64 114/172 2024-08-14T06:51:06.394 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-osd-2:19.0.0-5779.g488c4d2c.el9.x86_64 114/172 2024-08-14T06:51:06.394 INFO:teuthology.orchestra.run.smithi098.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:51:06.394 INFO:teuthology.orchestra.run.smithi098.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-14T06:51:06.394 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-14T06:51:06.395 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-14T06:51:06.395 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:06.587 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : cephfs-top-2:19.0.0-5779.g488c4d2c.el9.noarch 156/172 2024-08-14T06:51:06.618 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mds-2:19.0.0-5779.g488c4d2c.el9.x86_64 115/172 2024-08-14T06:51:06.641 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mds-2:19.0.0-5779.g488c4d2c.el9.x86_64 115/172 2024-08-14T06:51:06.641 INFO:teuthology.orchestra.run.smithi098.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:51:06.641 INFO:teuthology.orchestra.run.smithi098.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-14T06:51:06.641 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-14T06:51:06.641 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-14T06:51:06.641 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:06.691 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : openssl-devel-1:3.2.2-2.el9.x86_64 157/172 2024-08-14T06:51:06.732 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 158/172 2024-08-14T06:51:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : flex-2.6.4-9.el9.x86_64 159/172 2024-08-14T06:51:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : bison-3.7.4-5.el9.x86_64 160/172 2024-08-14T06:51:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : NetworkManager-tui-1:1.47.4-1.el9.x86_64 161/172 2024-08-14T06:51:06.912 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : elfutils-debuginfod-client-0.190-2.el9.x86_64 162/172 2024-08-14T06:51:06.938 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : elfutils-libs-0.190-2.el9.x86_64 163/172 2024-08-14T06:51:06.939 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 164/172 2024-08-14T06:51:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 164/172 2024-08-14T06:51:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : openssl-1:3.0.7-27.el9.x86_64 165/172 2024-08-14T06:51:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 166/172 2024-08-14T06:51:07.102 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mon-2:19.0.0-5779.g488c4d2c.el9.x86_64 116/172 2024-08-14T06:51:07.125 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mon-2:19.0.0-5779.g488c4d2c.el9.x86_64 116/172 2024-08-14T06:51:07.125 INFO:teuthology.orchestra.run.smithi098.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:51:07.125 INFO:teuthology.orchestra.run.smithi098.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-14T06:51:07.125 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-14T06:51:07.125 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-14T06:51:07.125 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:07.132 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 166/172 2024-08-14T06:51:07.157 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : NetworkManager-team-1:1.47.4-1.el9.x86_64 167/172 2024-08-14T06:51:07.163 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: NetworkManager-1:1.47.4-1.el9.x86_64 168/172 2024-08-14T06:51:07.163 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : NetworkManager-1:1.47.4-1.el9.x86_64 168/172 2024-08-14T06:51:07.241 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : automake-1.16.2-8.el9.noarch 117/172 2024-08-14T06:51:07.298 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-ply-3.11-14.el9.noarch 118/172 2024-08-14T06:51:07.353 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: NetworkManager-1:1.47.4-1.el9.x86_64 168/172 2024-08-14T06:51:07.353 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : NetworkManager-libnm-1:1.47.4-1.el9.x86_64 169/172 2024-08-14T06:51:07.362 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 119/172 2024-08-14T06:51:07.429 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.4-1.el9.x86_64 169/172 2024-08-14T06:51:07.453 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : openssl-libs-1:3.0.7-27.el9.x86_64 170/172 2024-08-14T06:51:07.484 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : elfutils-libelf-0.190-2.el9.x86_64 171/172 2024-08-14T06:51:07.547 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 120/172 2024-08-14T06:51:07.561 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : xfsprogs-6.3.0-1.el9.x86_64 172/172 2024-08-14T06:51:07.595 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 121/172 2024-08-14T06:51:07.665 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 122/172 2024-08-14T06:51:07.753 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 123/172 2024-08-14T06:51:07.885 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 124/172 2024-08-14T06:51:07.928 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 125/172 2024-08-14T06:51:07.961 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 126/172 2024-08-14T06:51:07.988 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : fuse-2.9.9-16.el9.x86_64 127/172 2024-08-14T06:51:08.451 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: cephadm-2:19.0.0-5779.g488c4d2c.el9.noarch 128/172 2024-08-14T06:51:08.479 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : cephadm-2:19.0.0-5779.g488c4d2c.el9.noarch 128/172 2024-08-14T06:51:08.512 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-5779.g488c4d2c.e 129/172 2024-08-14T06:51:08.567 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-5779.g488c4d2c. 130/172 2024-08-14T06:51:08.593 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-5779.g488c4d2c.el9.noa 131/172 2024-08-14T06:51:11.770 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-5779.g488c4d2c.el9.noa 131/172 2024-08-14T06:51:11.842 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-5779.g488c4d2c.el9.n 132/172 2024-08-14T06:51:12.796 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-5779.g488c4d2c.el9.n 132/172 2024-08-14T06:51:12.825 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-5779.g488 133/172 2024-08-14T06:51:12.938 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-5779.g488 133/172 2024-08-14T06:51:13.108 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-5779.g488c4d2c.el 134/172 2024-08-14T06:51:13.144 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-2:19.0.0-5779.g488c4d2c.el9.x86_64 135/172 2024-08-14T06:51:13.167 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mgr-2:19.0.0-5779.g488c4d2c.el9.x86_64 135/172 2024-08-14T06:51:13.167 INFO:teuthology.orchestra.run.smithi098.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:51:13.168 INFO:teuthology.orchestra.run.smithi098.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-14T06:51:13.168 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-14T06:51:13.168 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-14T06:51:13.168 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:13.211 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-rook-2:19.0.0-5779.g488c4d2c.el9.noarch 136/172 2024-08-14T06:51:13.222 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-5779.g488c4d2c.el9.noarch 136/172 2024-08-14T06:51:13.322 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-2:19.0.0-5779.g488c4d2c.el9.x86_64 137/172 2024-08-14T06:51:13.375 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 138/172 2024-08-14T06:51:14.349 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libtool-2.4.6-46.el9.x86_64 139/172 2024-08-14T06:51:14.381 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-radosgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 140/172 2024-08-14T06:51:14.403 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 140/172 2024-08-14T06:51:14.404 INFO:teuthology.orchestra.run.smithi098.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:51:14.404 INFO:teuthology.orchestra.run.smithi098.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-14T06:51:14.404 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-14T06:51:14.404 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-14T06:51:14.404 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:14.448 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-5779.g488c4 141/172 2024-08-14T06:51:14.470 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-5779.g488c4 141/172 2024-08-14T06:51:14.470 INFO:teuthology.orchestra.run.smithi098.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:51:14.470 INFO:teuthology.orchestra.run.smithi098.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-14T06:51:14.470 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:14.487 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: xfsprogs-6.3.0-1.el9.x86_64 172/172 2024-08-14T06:51:14.487 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-2:19.0.0-5779.g488c4d2c.el9.x86_64 1/172 2024-08-14T06:51:14.488 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-base-2:19.0.0-5779.g488c4d2c.el9.x86_64 2/172 2024-08-14T06:51:14.488 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 3/172 2024-08-14T06:51:14.488 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 4/172 2024-08-14T06:51:14.488 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-5779.g488c4 5/172 2024-08-14T06:51:14.488 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mds-2:19.0.0-5779.g488c4d2c.el9.x86_64 6/172 2024-08-14T06:51:14.488 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-2:19.0.0-5779.g488c4d2c.el9.x86_64 7/172 2024-08-14T06:51:14.488 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mon-2:19.0.0-5779.g488c4d2c.el9.x86_64 8/172 2024-08-14T06:51:14.488 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-osd-2:19.0.0-5779.g488c4d2c.el9.x86_64 9/172 2024-08-14T06:51:14.488 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-radosgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 10/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-selinux-2:19.0.0-5779.g488c4d2c.el9.x86_64 11/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-test-2:19.0.0-5779.g488c4d2c.el9.x86_64 12/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : cephfs-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 13/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : cephfs-shell-2:19.0.0-5779.g488c4d2c.el9.x86_64 14/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephfs-devel-2:19.0.0-5779.g488c4d2c.el9.x86_ 15/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephfs2-2:19.0.0-5779.g488c4d2c.el9.x86_64 16/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephsqlite-2:19.0.0-5779.g488c4d2c.el9.x86_64 17/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados-devel-2:19.0.0-5779.g488c4d2c.el9.x86_6 18/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libradosstriper1-2:19.0.0-5779.g488c4d2c.el9.x86 19/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librgw2-2:19.0.0-5779.g488c4d2c.el9.x86_64 20/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-ceph-argparse-2:19.0.0-5779.g488c4d2c.el 21/172 2024-08-14T06:51:14.489 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-ceph-common-2:19.0.0-5779.g488c4d2c.el9. 22/172 2024-08-14T06:51:14.490 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cephfs-2:19.0.0-5779.g488c4d2c.el9.x86_6 23/172 2024-08-14T06:51:14.490 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rados-2:19.0.0-5779.g488c4d2c.el9.x86_64 24/172 2024-08-14T06:51:14.490 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 25/172 2024-08-14T06:51:14.490 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 26/172 2024-08-14T06:51:14.490 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 27/172 2024-08-14T06:51:14.490 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 28/172 2024-08-14T06:51:14.490 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-nbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 29/172 2024-08-14T06:51:14.502 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-5779.g488c4d2c. 30/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-5779.g488c4d2c.el9.noa 31/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-5779.g488c4d2c.el9.n 32/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-5779.g488 33/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-5779.g488c4d2c.el 34/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-rook-2:19.0.0-5779.g488c4d2c.el9.noarch 35/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-5779.g488c4d2c.e 36/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-volume-2:19.0.0-5779.g488c4d2c.el9.noarch 37/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : cephadm-2:19.0.0-5779.g488c4d2c.el9.noarch 38/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : cephfs-top-2:19.0.0-5779.g488c4d2c.el9.noarch 39/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 40/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 41/172 2024-08-14T06:51:14.503 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 42/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 43/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 44/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-ply-3.11-14.el9.noarch 45/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 46/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 47/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 48/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : automake-1.16.2-8.el9.noarch 49/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : bison-3.7.4-5.el9.x86_64 50/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 51/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : elfutils-libelf-devel-0.191-4.el9.x86_64 52/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : flex-2.6.4-9.el9.x86_64 53/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 54/172 2024-08-14T06:51:14.504 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 55/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 56/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 57/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 58/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 59/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 60/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libtool-2.4.6-46.el9.x86_64 61/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 62/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 63/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 64/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : lua-5.4.4-4.el9.x86_64 65/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 66/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 67/172 2024-08-14T06:51:14.505 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openssl-devel-1:3.2.2-2.el9.x86_64 68/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : postgresql-13.14-1.el9.x86_64 69/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : postgresql-contrib-13.14-1.el9.x86_64 70/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : postgresql-private-libs-13.14-1.el9.x86_64 71/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : postgresql-server-13.14-1.el9.x86_64 72/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 73/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 74/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 75/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-colorama-0.4.6-3.el9.noarch 76/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 77/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 78/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 79/172 2024-08-14T06:51:14.506 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 80/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 81/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 82/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 83/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 84/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 85/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 86/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 87/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 88/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 89/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 90/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 91/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 92/172 2024-08-14T06:51:14.507 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 93/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 94/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 95/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 96/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 97/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 98/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 99/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 100/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : xfsprogs-devel-6.4.0-3.el9.x86_64 101/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 102/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : zlib-devel-1.2.11-41.el9.x86_64 103/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 104/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 105/172 2024-08-14T06:51:14.508 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 106/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 107/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 108/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 109/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 110/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 111/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 112/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 113/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 114/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 115/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 116/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 117/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 118/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 119/172 2024-08-14T06:51:14.509 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 120/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 121/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-google-auth-1:2.32.0-1.el9.noarch 122/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 123/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 124/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 125/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 126/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 127/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 128/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 129/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 130/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 131/172 2024-08-14T06:51:14.510 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 132/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 133/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 134/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 135/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 136/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 137/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 138/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 139/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 140/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 141/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 142/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 143/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 144/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 145/172 2024-08-14T06:51:14.511 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 146/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : re2-1:20211101-3.el9.x86_64 147/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 148/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados2-2:19.0.0-5779.g488c4d2c.el9.x86_64 149/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 150/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librbd1-2:19.0.0-5779.g488c4d2c.el9.x86_64 151/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 152/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : NetworkManager-1:1.48.2-2.el9.x86_64 153/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : NetworkManager-1:1.47.4-1.el9.x86_64 154/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : NetworkManager-libnm-1:1.48.2-2.el9.x86_64 155/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : NetworkManager-libnm-1:1.47.4-1.el9.x86_64 156/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : NetworkManager-team-1:1.48.2-2.el9.x86_64 157/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : NetworkManager-team-1:1.47.4-1.el9.x86_64 158/172 2024-08-14T06:51:14.512 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : NetworkManager-tui-1:1.48.2-2.el9.x86_64 159/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : NetworkManager-tui-1:1.47.4-1.el9.x86_64 160/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : elfutils-debuginfod-client-0.191-4.el9.x86_64 161/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 162/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : elfutils-libelf-0.191-4.el9.x86_64 163/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 164/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : elfutils-libs-0.191-4.el9.x86_64 165/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 166/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openssl-1:3.2.2-2.el9.x86_64 167/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 168/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openssl-libs-1:3.2.2-2.el9.x86_64 169/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 170/172 2024-08-14T06:51:14.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : xfsprogs-6.4.0-3.el9.x86_64 171/172 2024-08-14T06:51:14.515 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : cephfs-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 142/172 2024-08-14T06:51:14.538 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 142/172 2024-08-14T06:51:14.538 INFO:teuthology.orchestra.run.smithi098.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:51:14.538 INFO:teuthology.orchestra.run.smithi098.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-08-14T06:51:14.538 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-08-14T06:51:14.538 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-08-14T06:51:14.538 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:14.866 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : rbd-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 143/172 2024-08-14T06:51:14.888 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: rbd-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 143/172 2024-08-14T06:51:14.889 INFO:teuthology.orchestra.run.smithi098.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-14T06:51:14.889 INFO:teuthology.orchestra.run.smithi098.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-14T06:51:14.889 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-14T06:51:14.889 INFO:teuthology.orchestra.run.smithi098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-14T06:51:14.889 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:17.231 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : xfsprogs-6.3.0-1.el9.x86_64 172/172 2024-08-14T06:51:17.231 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:17.231 INFO:teuthology.orchestra.run.smithi191.stdout:Upgraded: 2024-08-14T06:51:17.231 INFO:teuthology.orchestra.run.smithi191.stdout: NetworkManager-1:1.48.2-2.el9.x86_64 2024-08-14T06:51:17.231 INFO:teuthology.orchestra.run.smithi191.stdout: NetworkManager-libnm-1:1.48.2-2.el9.x86_64 2024-08-14T06:51:17.231 INFO:teuthology.orchestra.run.smithi191.stdout: NetworkManager-team-1:1.48.2-2.el9.x86_64 2024-08-14T06:51:17.231 INFO:teuthology.orchestra.run.smithi191.stdout: NetworkManager-tui-1:1.48.2-2.el9.x86_64 2024-08-14T06:51:17.231 INFO:teuthology.orchestra.run.smithi191.stdout: elfutils-debuginfod-client-0.191-4.el9.x86_64 2024-08-14T06:51:17.231 INFO:teuthology.orchestra.run.smithi191.stdout: elfutils-libelf-0.191-4.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: elfutils-libs-0.191-4.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: librados2-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: librbd1-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: openssl-1:3.2.2-2.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: openssl-libs-1:3.2.2-2.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: xfsprogs-6.4.0-3.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: automake-1.16.2-8.el9.noarch 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: bison-3.7.4-5.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.232 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-base-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-grafana-dashboards-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-immutable-object-cache-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mds-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-cephadm-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-dashboard-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-diskprediction-local-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-modules-core-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-rook-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:17.233 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mon-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-osd-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-prometheus-alerts-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-radosgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-selinux-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-test-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-volume-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: cephadm-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: cephfs-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: cephfs-shell-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: cephfs-top-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: elfutils-libelf-devel-0.191-4.el9.x86_64 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: flex-2.6.4-9.el9.x86_64 2024-08-14T06:51:17.234 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: fuse-2.9.9-16.el9.x86_64 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: grpc-data-1.46.7-10.el9.noarch 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: libarrow-9.0.0-11.el9.x86_64 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs-devel-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs2-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: libcephsqlite-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.235 INFO:teuthology.orchestra.run.smithi191.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: librados-devel-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: libradosstriper1-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: librgw2-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: libtool-2.4.6-46.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-14T06:51:17.236 INFO:teuthology.orchestra.run.smithi191.stdout: lua-5.4.4-4.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: luarocks-3.9.2-5.el9.noarch 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-0.3.21-2.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: openssl-devel-1:3.2.2-2.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: postgresql-13.14-1.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: postgresql-contrib-13.14-1.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: postgresql-private-libs-13.14-1.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: postgresql-server-13.14-1.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: protobuf-3.14.0-13.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-08-14T06:51:17.237 INFO:teuthology.orchestra.run.smithi191.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-14T06:51:17.238 INFO:teuthology.orchestra.run.smithi191.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-08-14T06:51:17.238 INFO:teuthology.orchestra.run.smithi191.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-14T06:51:17.238 INFO:teuthology.orchestra.run.smithi191.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-14T06:51:17.238 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-14T06:51:17.238 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-argparse-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.238 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.238 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cephfs-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.238 INFO:teuthology.orchestra.run.smithi191.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-colorama-0.4.6-3.el9.noarch 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-google-auth-1:2.32.0-1.el9.noarch 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-14T06:51:17.239 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-14T06:51:17.240 INFO:teuthology.orchestra.run.smithi191.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-packaging-20.9-5.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ply-3.11-14.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-08-14T06:51:17.241 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rados-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-14T06:51:17.242 INFO:teuthology.orchestra.run.smithi191.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webob-1.8.7-6.el9.noarch 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: qatlib-24.02.0-1.el9.x86_64 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-nbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: re2-1:20211101-3.el9.x86_64 2024-08-14T06:51:17.243 INFO:teuthology.orchestra.run.smithi191.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-14T06:51:17.244 INFO:teuthology.orchestra.run.smithi191.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-14T06:51:17.244 INFO:teuthology.orchestra.run.smithi191.stdout: uuid-1.6.2-55.el9.x86_64 2024-08-14T06:51:17.244 INFO:teuthology.orchestra.run.smithi191.stdout: xfsprogs-devel-6.4.0-3.el9.x86_64 2024-08-14T06:51:17.244 INFO:teuthology.orchestra.run.smithi191.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-14T06:51:17.244 INFO:teuthology.orchestra.run.smithi191.stdout: zlib-devel-1.2.11-41.el9.x86_64 2024-08-14T06:51:17.244 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:17.244 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-08-14T06:51:17.709 DEBUG:teuthology.parallel:result is None 2024-08-14T06:51:19.237 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-test-2:19.0.0-5779.g488c4d2c.el9.x86_64 144/172 2024-08-14T06:51:19.286 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : rbd-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 145/172 2024-08-14T06:51:19.328 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : rbd-nbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 146/172 2024-08-14T06:51:19.359 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : xfsprogs-devel-6.4.0-3.el9.x86_64 147/172 2024-08-14T06:51:19.386 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : elfutils-libelf-devel-0.191-4.el9.x86_64 148/172 2024-08-14T06:51:19.585 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : cephfs-shell-2:19.0.0-5779.g488c4d2c.el9.x86_64 149/172 2024-08-14T06:51:19.624 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : postgresql-contrib-13.14-1.el9.x86_64 150/172 2024-08-14T06:51:19.662 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libcephfs-devel-2:19.0.0-5779.g488c4d2c.el9.x86_ 151/172 2024-08-14T06:51:19.697 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : elfutils-debuginfod-client-0.191-4.el9.x86_64 152/172 2024-08-14T06:51:19.727 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : NetworkManager-team-1:1.48.2-2.el9.x86_64 153/172 2024-08-14T06:51:19.765 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : NetworkManager-tui-1:1.48.2-2.el9.x86_64 154/172 2024-08-14T06:51:20.407 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 155/172 2024-08-14T06:51:20.452 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : postgresql-server-13.14-1.el9.x86_64 155/172 2024-08-14T06:51:20.478 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 155/172 2024-08-14T06:51:21.310 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : cephfs-top-2:19.0.0-5779.g488c4d2c.el9.noarch 156/172 2024-08-14T06:51:21.410 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : openssl-devel-1:3.2.2-2.el9.x86_64 157/172 2024-08-14T06:51:21.451 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 158/172 2024-08-14T06:51:21.549 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : flex-2.6.4-9.el9.x86_64 159/172 2024-08-14T06:51:21.586 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : bison-3.7.4-5.el9.x86_64 160/172 2024-08-14T06:51:21.611 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : NetworkManager-tui-1:1.47.4-1.el9.x86_64 161/172 2024-08-14T06:51:21.637 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : elfutils-debuginfod-client-0.190-2.el9.x86_64 162/172 2024-08-14T06:51:21.663 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : elfutils-libs-0.190-2.el9.x86_64 163/172 2024-08-14T06:51:21.663 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 164/172 2024-08-14T06:51:21.756 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 164/172 2024-08-14T06:51:21.789 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : openssl-1:3.0.7-27.el9.x86_64 165/172 2024-08-14T06:51:21.789 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 166/172 2024-08-14T06:51:21.915 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 166/172 2024-08-14T06:51:21.940 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : NetworkManager-team-1:1.47.4-1.el9.x86_64 167/172 2024-08-14T06:51:21.946 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: NetworkManager-1:1.47.4-1.el9.x86_64 168/172 2024-08-14T06:51:21.946 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : NetworkManager-1:1.47.4-1.el9.x86_64 168/172 2024-08-14T06:51:22.126 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: NetworkManager-1:1.47.4-1.el9.x86_64 168/172 2024-08-14T06:51:22.126 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : NetworkManager-libnm-1:1.47.4-1.el9.x86_64 169/172 2024-08-14T06:51:22.202 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.4-1.el9.x86_64 169/172 2024-08-14T06:51:22.227 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : openssl-libs-1:3.0.7-27.el9.x86_64 170/172 2024-08-14T06:51:22.260 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : elfutils-libelf-0.190-2.el9.x86_64 171/172 2024-08-14T06:51:22.328 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : xfsprogs-6.3.0-1.el9.x86_64 172/172 2024-08-14T06:51:29.005 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: xfsprogs-6.3.0-1.el9.x86_64 172/172 2024-08-14T06:51:29.005 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-2:19.0.0-5779.g488c4d2c.el9.x86_64 1/172 2024-08-14T06:51:29.005 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-base-2:19.0.0-5779.g488c4d2c.el9.x86_64 2/172 2024-08-14T06:51:29.005 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 3/172 2024-08-14T06:51:29.005 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 4/172 2024-08-14T06:51:29.006 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-5779.g488c4 5/172 2024-08-14T06:51:29.006 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mds-2:19.0.0-5779.g488c4d2c.el9.x86_64 6/172 2024-08-14T06:51:29.006 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-2:19.0.0-5779.g488c4d2c.el9.x86_64 7/172 2024-08-14T06:51:29.006 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mon-2:19.0.0-5779.g488c4d2c.el9.x86_64 8/172 2024-08-14T06:51:29.006 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-osd-2:19.0.0-5779.g488c4d2c.el9.x86_64 9/172 2024-08-14T06:51:29.006 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-radosgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 10/172 2024-08-14T06:51:29.006 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-selinux-2:19.0.0-5779.g488c4d2c.el9.x86_64 11/172 2024-08-14T06:51:29.006 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-test-2:19.0.0-5779.g488c4d2c.el9.x86_64 12/172 2024-08-14T06:51:29.006 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : cephfs-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 13/172 2024-08-14T06:51:29.006 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : cephfs-shell-2:19.0.0-5779.g488c4d2c.el9.x86_64 14/172 2024-08-14T06:51:29.007 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libcephfs-devel-2:19.0.0-5779.g488c4d2c.el9.x86_ 15/172 2024-08-14T06:51:29.007 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libcephfs2-2:19.0.0-5779.g488c4d2c.el9.x86_64 16/172 2024-08-14T06:51:29.007 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libcephsqlite-2:19.0.0-5779.g488c4d2c.el9.x86_64 17/172 2024-08-14T06:51:29.007 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librados-devel-2:19.0.0-5779.g488c4d2c.el9.x86_6 18/172 2024-08-14T06:51:29.007 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libradosstriper1-2:19.0.0-5779.g488c4d2c.el9.x86 19/172 2024-08-14T06:51:29.007 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librgw2-2:19.0.0-5779.g488c4d2c.el9.x86_64 20/172 2024-08-14T06:51:29.007 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-ceph-argparse-2:19.0.0-5779.g488c4d2c.el 21/172 2024-08-14T06:51:29.007 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-ceph-common-2:19.0.0-5779.g488c4d2c.el9. 22/172 2024-08-14T06:51:29.007 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cephfs-2:19.0.0-5779.g488c4d2c.el9.x86_6 23/172 2024-08-14T06:51:29.007 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-rados-2:19.0.0-5779.g488c4d2c.el9.x86_64 24/172 2024-08-14T06:51:29.008 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-rbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 25/172 2024-08-14T06:51:29.008 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-rgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 26/172 2024-08-14T06:51:29.008 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : rbd-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 27/172 2024-08-14T06:51:29.008 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : rbd-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 28/172 2024-08-14T06:51:29.008 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : rbd-nbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 29/172 2024-08-14T06:51:29.020 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-5779.g488c4d2c. 30/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-5779.g488c4d2c.el9.noa 31/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-5779.g488c4d2c.el9.n 32/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-5779.g488 33/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-5779.g488c4d2c.el 34/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-rook-2:19.0.0-5779.g488c4d2c.el9.noarch 35/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-5779.g488c4d2c.e 36/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-volume-2:19.0.0-5779.g488c4d2c.el9.noarch 37/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : cephadm-2:19.0.0-5779.g488c4d2c.el9.noarch 38/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : cephfs-top-2:19.0.0-5779.g488c4d2c.el9.noarch 39/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 40/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 41/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 42/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 43/172 2024-08-14T06:51:29.021 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 44/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-ply-3.11-14.el9.noarch 45/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 46/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 47/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 48/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : automake-1.16.2-8.el9.noarch 49/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : bison-3.7.4-5.el9.x86_64 50/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 51/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : elfutils-libelf-devel-0.191-4.el9.x86_64 52/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : flex-2.6.4-9.el9.x86_64 53/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 54/172 2024-08-14T06:51:29.022 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 55/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 56/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 57/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 58/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 59/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 60/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libtool-2.4.6-46.el9.x86_64 61/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 62/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 63/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 64/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : lua-5.4.4-4.el9.x86_64 65/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 66/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 67/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : openssl-devel-1:3.2.2-2.el9.x86_64 68/172 2024-08-14T06:51:29.023 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : postgresql-13.14-1.el9.x86_64 69/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : postgresql-contrib-13.14-1.el9.x86_64 70/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : postgresql-private-libs-13.14-1.el9.x86_64 71/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : postgresql-server-13.14-1.el9.x86_64 72/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 73/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 74/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 75/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-colorama-0.4.6-3.el9.noarch 76/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 77/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 78/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 79/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 80/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 81/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 82/172 2024-08-14T06:51:29.024 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 83/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 84/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 85/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 86/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 87/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 88/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 89/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 90/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 91/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 92/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 93/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 94/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 95/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 96/172 2024-08-14T06:51:29.025 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 97/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 98/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 99/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 100/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : xfsprogs-devel-6.4.0-3.el9.x86_64 101/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 102/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : zlib-devel-1.2.11-41.el9.x86_64 103/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 104/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 105/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 106/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 107/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 108/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 109/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 110/172 2024-08-14T06:51:29.026 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 111/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 112/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 113/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 114/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 115/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 116/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 117/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 118/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 119/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 120/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 121/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-google-auth-1:2.32.0-1.el9.noarch 122/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 123/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 124/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 125/172 2024-08-14T06:51:29.027 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 126/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 127/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 128/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 129/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 130/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 131/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 132/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 133/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 134/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 135/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 136/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 137/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 138/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 139/172 2024-08-14T06:51:29.028 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 140/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 141/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 142/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 143/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 144/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 145/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 146/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : re2-1:20211101-3.el9.x86_64 147/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 148/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librados2-2:19.0.0-5779.g488c4d2c.el9.x86_64 149/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 150/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librbd1-2:19.0.0-5779.g488c4d2c.el9.x86_64 151/172 2024-08-14T06:51:29.029 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 152/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : NetworkManager-1:1.48.2-2.el9.x86_64 153/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : NetworkManager-1:1.47.4-1.el9.x86_64 154/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : NetworkManager-libnm-1:1.48.2-2.el9.x86_64 155/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : NetworkManager-libnm-1:1.47.4-1.el9.x86_64 156/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : NetworkManager-team-1:1.48.2-2.el9.x86_64 157/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : NetworkManager-team-1:1.47.4-1.el9.x86_64 158/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : NetworkManager-tui-1:1.48.2-2.el9.x86_64 159/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : NetworkManager-tui-1:1.47.4-1.el9.x86_64 160/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : elfutils-debuginfod-client-0.191-4.el9.x86_64 161/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 162/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : elfutils-libelf-0.191-4.el9.x86_64 163/172 2024-08-14T06:51:29.030 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 164/172 2024-08-14T06:51:29.031 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : elfutils-libs-0.191-4.el9.x86_64 165/172 2024-08-14T06:51:29.031 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 166/172 2024-08-14T06:51:29.031 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : openssl-1:3.2.2-2.el9.x86_64 167/172 2024-08-14T06:51:29.031 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 168/172 2024-08-14T06:51:29.031 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : openssl-libs-1:3.2.2-2.el9.x86_64 169/172 2024-08-14T06:51:29.031 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 170/172 2024-08-14T06:51:29.031 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : xfsprogs-6.4.0-3.el9.x86_64 171/172 2024-08-14T06:51:31.636 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : xfsprogs-6.3.0-1.el9.x86_64 172/172 2024-08-14T06:51:31.637 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:31.637 INFO:teuthology.orchestra.run.smithi098.stdout:Upgraded: 2024-08-14T06:51:31.637 INFO:teuthology.orchestra.run.smithi098.stdout: NetworkManager-1:1.48.2-2.el9.x86_64 2024-08-14T06:51:31.637 INFO:teuthology.orchestra.run.smithi098.stdout: NetworkManager-libnm-1:1.48.2-2.el9.x86_64 2024-08-14T06:51:31.637 INFO:teuthology.orchestra.run.smithi098.stdout: NetworkManager-team-1:1.48.2-2.el9.x86_64 2024-08-14T06:51:31.637 INFO:teuthology.orchestra.run.smithi098.stdout: NetworkManager-tui-1:1.48.2-2.el9.x86_64 2024-08-14T06:51:31.638 INFO:teuthology.orchestra.run.smithi098.stdout: elfutils-debuginfod-client-0.191-4.el9.x86_64 2024-08-14T06:51:31.638 INFO:teuthology.orchestra.run.smithi098.stdout: elfutils-libelf-0.191-4.el9.x86_64 2024-08-14T06:51:31.638 INFO:teuthology.orchestra.run.smithi098.stdout: elfutils-libs-0.191-4.el9.x86_64 2024-08-14T06:51:31.638 INFO:teuthology.orchestra.run.smithi098.stdout: librados2-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.638 INFO:teuthology.orchestra.run.smithi098.stdout: librbd1-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.639 INFO:teuthology.orchestra.run.smithi098.stdout: openssl-1:3.2.2-2.el9.x86_64 2024-08-14T06:51:31.639 INFO:teuthology.orchestra.run.smithi098.stdout: openssl-libs-1:3.2.2-2.el9.x86_64 2024-08-14T06:51:31.639 INFO:teuthology.orchestra.run.smithi098.stdout: xfsprogs-6.4.0-3.el9.x86_64 2024-08-14T06:51:31.639 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2024-08-14T06:51:31.639 INFO:teuthology.orchestra.run.smithi098.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-08-14T06:51:31.639 INFO:teuthology.orchestra.run.smithi098.stdout: automake-1.16.2-8.el9.noarch 2024-08-14T06:51:31.639 INFO:teuthology.orchestra.run.smithi098.stdout: bison-3.7.4-5.el9.x86_64 2024-08-14T06:51:31.640 INFO:teuthology.orchestra.run.smithi098.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-08-14T06:51:31.640 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.640 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-base-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.640 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.640 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.641 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-grafana-dashboards-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:31.641 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-immutable-object-cache-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.641 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mds-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.641 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.641 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-cephadm-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:31.641 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-dashboard-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:31.641 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-diskprediction-local-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:31.642 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-modules-core-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:31.642 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-rook-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:31.642 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mon-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.642 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-osd-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.642 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-prometheus-alerts-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:31.643 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-radosgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.643 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-selinux-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.643 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-test-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.643 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-volume-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:31.643 INFO:teuthology.orchestra.run.smithi098.stdout: cephadm-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:31.643 INFO:teuthology.orchestra.run.smithi098.stdout: cephfs-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.644 INFO:teuthology.orchestra.run.smithi098.stdout: cephfs-shell-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.644 INFO:teuthology.orchestra.run.smithi098.stdout: cephfs-top-2:19.0.0-5779.g488c4d2c.el9.noarch 2024-08-14T06:51:31.644 INFO:teuthology.orchestra.run.smithi098.stdout: elfutils-libelf-devel-0.191-4.el9.x86_64 2024-08-14T06:51:31.644 INFO:teuthology.orchestra.run.smithi098.stdout: flex-2.6.4-9.el9.x86_64 2024-08-14T06:51:31.644 INFO:teuthology.orchestra.run.smithi098.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-14T06:51:31.644 INFO:teuthology.orchestra.run.smithi098.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-14T06:51:31.645 INFO:teuthology.orchestra.run.smithi098.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-14T06:51:31.645 INFO:teuthology.orchestra.run.smithi098.stdout: fuse-2.9.9-16.el9.x86_64 2024-08-14T06:51:31.645 INFO:teuthology.orchestra.run.smithi098.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-14T06:51:31.645 INFO:teuthology.orchestra.run.smithi098.stdout: grpc-data-1.46.7-10.el9.noarch 2024-08-14T06:51:31.645 INFO:teuthology.orchestra.run.smithi098.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-08-14T06:51:31.645 INFO:teuthology.orchestra.run.smithi098.stdout: libarrow-9.0.0-11.el9.x86_64 2024-08-14T06:51:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-08-14T06:51:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: libcephfs-devel-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: libcephfs2-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: libcephsqlite-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-08-14T06:51:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-14T06:51:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-14T06:51:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-08-14T06:51:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-14T06:51:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: librados-devel-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: libradosstriper1-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-14T06:51:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: librgw2-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: libtool-2.4.6-46.el9.x86_64 2024-08-14T06:51:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-14T06:51:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-08-14T06:51:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-08-14T06:51:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-14T06:51:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: lua-5.4.4-4.el9.x86_64 2024-08-14T06:51:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-08-14T06:51:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: luarocks-3.9.2-5.el9.noarch 2024-08-14T06:51:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: openblas-0.3.21-2.el9.x86_64 2024-08-14T06:51:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-08-14T06:51:31.650 INFO:teuthology.orchestra.run.smithi098.stdout: openssl-devel-1:3.2.2-2.el9.x86_64 2024-08-14T06:51:31.650 INFO:teuthology.orchestra.run.smithi098.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-08-14T06:51:31.650 INFO:teuthology.orchestra.run.smithi098.stdout: postgresql-13.14-1.el9.x86_64 2024-08-14T06:51:31.650 INFO:teuthology.orchestra.run.smithi098.stdout: postgresql-contrib-13.14-1.el9.x86_64 2024-08-14T06:51:31.650 INFO:teuthology.orchestra.run.smithi098.stdout: postgresql-private-libs-13.14-1.el9.x86_64 2024-08-14T06:51:31.650 INFO:teuthology.orchestra.run.smithi098.stdout: postgresql-server-13.14-1.el9.x86_64 2024-08-14T06:51:31.651 INFO:teuthology.orchestra.run.smithi098.stdout: protobuf-3.14.0-13.el9.x86_64 2024-08-14T06:51:31.651 INFO:teuthology.orchestra.run.smithi098.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-08-14T06:51:31.651 INFO:teuthology.orchestra.run.smithi098.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-14T06:51:31.651 INFO:teuthology.orchestra.run.smithi098.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-08-14T06:51:31.651 INFO:teuthology.orchestra.run.smithi098.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-14T06:51:31.651 INFO:teuthology.orchestra.run.smithi098.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-14T06:51:31.652 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-14T06:51:31.652 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ceph-argparse-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.652 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ceph-common-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.652 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cephfs-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.652 INFO:teuthology.orchestra.run.smithi098.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-14T06:51:31.652 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-14T06:51:31.653 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-14T06:51:31.653 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-14T06:51:31.653 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-08-14T06:51:31.653 INFO:teuthology.orchestra.run.smithi098.stdout: python3-colorama-0.4.6-3.el9.noarch 2024-08-14T06:51:31.653 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-14T06:51:31.653 INFO:teuthology.orchestra.run.smithi098.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-08-14T06:51:31.654 INFO:teuthology.orchestra.run.smithi098.stdout: python3-google-auth-1:2.32.0-1.el9.noarch 2024-08-14T06:51:31.654 INFO:teuthology.orchestra.run.smithi098.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-08-14T06:51:31.654 INFO:teuthology.orchestra.run.smithi098.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-08-14T06:51:31.654 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-14T06:51:31.654 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-14T06:51:31.654 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-14T06:51:31.655 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-14T06:51:31.655 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-14T06:51:31.655 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-14T06:51:31.655 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-14T06:51:31.655 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-14T06:51:31.655 INFO:teuthology.orchestra.run.smithi098.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-14T06:51:31.656 INFO:teuthology.orchestra.run.smithi098.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-14T06:51:31.656 INFO:teuthology.orchestra.run.smithi098.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-14T06:51:31.656 INFO:teuthology.orchestra.run.smithi098.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-14T06:51:31.656 INFO:teuthology.orchestra.run.smithi098.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-14T06:51:31.656 INFO:teuthology.orchestra.run.smithi098.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-14T06:51:31.656 INFO:teuthology.orchestra.run.smithi098.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-14T06:51:31.656 INFO:teuthology.orchestra.run.smithi098.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-14T06:51:31.657 INFO:teuthology.orchestra.run.smithi098.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-14T06:51:31.657 INFO:teuthology.orchestra.run.smithi098.stdout: python3-packaging-20.9-5.el9.noarch 2024-08-14T06:51:31.657 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-14T06:51:31.657 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ply-3.11-14.el9.noarch 2024-08-14T06:51:31.657 INFO:teuthology.orchestra.run.smithi098.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-14T06:51:31.657 INFO:teuthology.orchestra.run.smithi098.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-14T06:51:31.657 INFO:teuthology.orchestra.run.smithi098.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-08-14T06:51:31.658 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-14T06:51:31.658 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-14T06:51:31.658 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-14T06:51:31.658 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-14T06:51:31.658 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-08-14T06:51:31.658 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-14T06:51:31.658 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rados-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.659 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.659 INFO:teuthology.orchestra.run.smithi098.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-14T06:51:31.659 INFO:teuthology.orchestra.run.smithi098.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-14T06:51:31.659 INFO:teuthology.orchestra.run.smithi098.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-14T06:51:31.659 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rgw-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.659 INFO:teuthology.orchestra.run.smithi098.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-14T06:51:31.660 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-14T06:51:31.660 INFO:teuthology.orchestra.run.smithi098.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-14T06:51:31.660 INFO:teuthology.orchestra.run.smithi098.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-14T06:51:31.660 INFO:teuthology.orchestra.run.smithi098.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-14T06:51:31.660 INFO:teuthology.orchestra.run.smithi098.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-14T06:51:31.660 INFO:teuthology.orchestra.run.smithi098.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-14T06:51:31.661 INFO:teuthology.orchestra.run.smithi098.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-08-14T06:51:31.661 INFO:teuthology.orchestra.run.smithi098.stdout: python3-webob-1.8.7-6.el9.noarch 2024-08-14T06:51:31.661 INFO:teuthology.orchestra.run.smithi098.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-14T06:51:31.661 INFO:teuthology.orchestra.run.smithi098.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-14T06:51:31.661 INFO:teuthology.orchestra.run.smithi098.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-14T06:51:31.661 INFO:teuthology.orchestra.run.smithi098.stdout: qatlib-24.02.0-1.el9.x86_64 2024-08-14T06:51:31.661 INFO:teuthology.orchestra.run.smithi098.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-08-14T06:51:31.662 INFO:teuthology.orchestra.run.smithi098.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-08-14T06:51:31.662 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-fuse-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.662 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-mirror-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.662 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-nbd-2:19.0.0-5779.g488c4d2c.el9.x86_64 2024-08-14T06:51:31.662 INFO:teuthology.orchestra.run.smithi098.stdout: re2-1:20211101-3.el9.x86_64 2024-08-14T06:51:31.662 INFO:teuthology.orchestra.run.smithi098.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-14T06:51:31.662 INFO:teuthology.orchestra.run.smithi098.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-14T06:51:31.663 INFO:teuthology.orchestra.run.smithi098.stdout: uuid-1.6.2-55.el9.x86_64 2024-08-14T06:51:31.663 INFO:teuthology.orchestra.run.smithi098.stdout: xfsprogs-devel-6.4.0-3.el9.x86_64 2024-08-14T06:51:31.663 INFO:teuthology.orchestra.run.smithi098.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-14T06:51:31.663 INFO:teuthology.orchestra.run.smithi098.stdout: zlib-devel-1.2.11-41.el9.x86_64 2024-08-14T06:51:31.664 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:31.664 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2024-08-14T06:51:32.078 DEBUG:teuthology.parallel:result is None 2024-08-14T06:51:32.079 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=488c4d2c8097ced3f3bab85cfc6687562b3affe3 2024-08-14T06:51:32.243 DEBUG:teuthology.orchestra.run.smithi098:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-14T06:51:32.267 INFO:teuthology.orchestra.run.smithi098.stdout:19.0.0-5779.g488c4d2c.el9 2024-08-14T06:51:32.267 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-5779.g488c4d2c.el9 2024-08-14T06:51:32.267 INFO:teuthology.task.install:The correct ceph version 19.0.0-5779.g488c4d2c is installed. 2024-08-14T06:51:32.270 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=488c4d2c8097ced3f3bab85cfc6687562b3affe3 2024-08-14T06:51:32.418 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-14T06:51:32.445 INFO:teuthology.orchestra.run.smithi191.stdout:19.0.0-5779.g488c4d2c.el9 2024-08-14T06:51:32.445 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-5779.g488c4d2c.el9 2024-08-14T06:51:32.445 INFO:teuthology.task.install:The correct ceph version 19.0.0-5779.g488c4d2c is installed. 2024-08-14T06:51:32.447 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-08-14T06:51:32.447 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:32.447 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-14T06:51:32.477 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:51:32.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-14T06:51:32.516 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-08-14T06:51:32.517 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:32.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/usr/bin/daemon-helper 2024-08-14T06:51:32.544 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-14T06:51:32.610 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:51:32.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/daemon-helper 2024-08-14T06:51:32.638 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-14T06:51:32.703 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-08-14T06:51:32.704 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:32.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-14T06:51:32.729 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-14T06:51:32.795 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:51:32.795 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-14T06:51:32.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-14T06:51:32.886 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-08-14T06:51:32.886 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:32.886 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/usr/bin/stdin-killer 2024-08-14T06:51:32.911 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-14T06:51:32.976 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:51:32.977 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/stdin-killer 2024-08-14T06:51:33.003 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-14T06:51:33.068 INFO:teuthology.run_tasks:Running task ceph... 2024-08-14T06:51:33.166 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-08-14T06:51:33.169 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 777 /var/log/ceph 2024-08-14T06:51:33.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 777 /var/log/ceph 2024-08-14T06:51:33.197 INFO:tasks.ceph:Disabling ceph logrotate... 2024-08-14T06:51:33.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -f -- /etc/logrotate.d/ceph 2024-08-14T06:51:33.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo rm -f -- /etc/logrotate.d/ceph 2024-08-14T06:51:33.262 INFO:tasks.ceph:Creating extra log directories... 2024-08-14T06:51:33.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-08-14T06:51:33.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-08-14T06:51:33.333 INFO:tasks.ceph:Setting up log rotation with {'ceph-mds': '10G', 'ceph-osd': '10G'} 2024-08-14T06:51:33.333 INFO:tasks.ceph:writing logrotate stanza for ceph-mds 2024-08-14T06:51:33.333 INFO:tasks.ceph:writing logrotate stanza for ceph-osd 2024-08-14T06:51:33.333 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:33.334 DEBUG:teuthology.orchestra.run.smithi098:> dd of=/home/ubuntu/cephtest/logrotate.ceph-test.conf 2024-08-14T06:51:33.379 DEBUG:teuthology.orchestra.run.smithi098:> sudo mv /home/ubuntu/cephtest/logrotate.ceph-test.conf /etc/logrotate.d/ceph-test.conf && sudo chmod 0644 /etc/logrotate.d/ceph-test.conf && sudo chown root.root /etc/logrotate.d/ceph-test.conf 2024-08-14T06:51:33.496 DEBUG:teuthology.orchestra.run.smithi098:> sudo chcon system_u:object_r:etc_t:s0 /etc/logrotate.d/ceph-test.conf 2024-08-14T06:51:33.520 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:51:33.520 DEBUG:teuthology.orchestra.run.smithi191:> dd of=/home/ubuntu/cephtest/logrotate.ceph-test.conf 2024-08-14T06:51:33.536 DEBUG:teuthology.orchestra.run.smithi191:> sudo mv /home/ubuntu/cephtest/logrotate.ceph-test.conf /etc/logrotate.d/ceph-test.conf && sudo chmod 0644 /etc/logrotate.d/ceph-test.conf && sudo chown root.root /etc/logrotate.d/ceph-test.conf 2024-08-14T06:51:33.651 DEBUG:teuthology.orchestra.run.smithi191:> sudo chcon system_u:object_r:etc_t:s0 /etc/logrotate.d/ceph-test.conf 2024-08-14T06:51:33.680 INFO:tasks.ceph:Creating ceph cluster ceph... 2024-08-14T06:51:33.681 INFO:tasks.ceph:config {'conf': {'client': {'pid file': ''}, 'global': {'ms die on skipped message': False}, 'mgr': {'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'bad backtrace', 'object missing on disk', 'error reading table object', 'error reading sessionmap', 'unmatched fragstat', 'unmatched rstat', 'was unreadable, recreating it now', 'Scrub error on inode', 'Metadata damage detected', 'MDS_FAILED', 'MDS_DAMAGE'], 'cpu_profile': set(), 'cluster': 'ceph', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-08-14T06:51:33.681 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/pdonnell-2024-08-14_02:13:41-fs-wip-pdonnell-testing-20240813.125846-debug-distro-default-smithi/7854349', 'branch': 'wip-pdonnell-testing-20240813.125846-debug', 'description': 'fs/functional/{begin/{0-install 1-ceph 2-logrotate 3-modules} clusters/1a3s-mds-4c-client conf/{client mds mgr mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{centos_9.stream k-stock} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile pg_health} subvol_versions/create_subvol_version_v2 tasks/alternate-pool}', 'email': 'pdonnell@redhat.com', 'first_in_suite': False, 'job_id': '7854349', 'kernel': {'client': {'sha1': 'distro'}, 'kdb': 1, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'log-rotate': {'ceph-mds': '10G', 'ceph-osd': '10G'}, 'machine_type': 'smithi', 'mgrmodules': {'sequential': [{'print': 'Enabling mgr modules'}]}, 'name': 'pdonnell-2024-08-14_02:13:41-fs-wip-pdonnell-testing-20240813.125846-debug-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 4, 'size': 20}}, {'machine': {'disk': 200}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-pdonnell-testing-20240813.125846-debug'}, 'ceph': {'cluster-conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 2, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mon': {'mon op complaint time': 120, 'mon warn on pool no app': False}, 'osd': {'osd op complaint time': 180}}, 'conf': {'client': {'pid file': ''}, 'global': {'ms die on skipped message': False}, 'mgr': {'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'bad backtrace', 'object missing on disk', 'error reading table object', 'error reading sessionmap', 'unmatched fragstat', 'unmatched rstat', 'was unreadable, recreating it now', 'Scrub error on inode', 'Metadata damage detected', 'MDS_FAILED', 'MDS_DAMAGE'], 'sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3'}}, 'kclient': {'syntax': 'v2'}, 'subvolume_version': 2, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-pdonnell-testing-20240813.125846-debug', 'sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3'}}, 'owner': 'scheduled_pdonnell@teuthology', 'priority': 90, 'repo': 'https://git.ceph.com/ceph-ci.git', 'roles': [['mon.a', 'mgr.y', 'mds.a', 'mds.c', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'client.2'], ['mon.b', 'mon.c', 'mgr.x', 'mds.b', 'mds.d', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'client.3']], 'seed': 8073, 'sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3', 'sleep_before_teardown': 0, 'subset': '511/512', 'suite': 'fs', 'suite_branch': 'wip-pdonnell-testing-20240813.125846-debug', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph-c_488c4d2c8097ced3f3bab85cfc6687562b3affe3/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph-ci.git', 'suite_sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3', 'syslog': {'ignorelist': ['WARNING*.*check_session_state', 'WARNING*.*__ceph_remove_cap']}, 'targets': {'smithi098.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLEZbyx2EaBgUyzjuo7qSCWQMbTf+yHkY6kOaVcpALjZki/tddSd1y1TEuAJDn4q0YIK+oIzlGSC0ekfTswZKvo=', 'smithi191.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHAEnArlr9jbW6v7LWN48BdymRadcg88rHYqyEmE5nS8+HvlARffhhz9joA2m0wa7kXPJrl9aRLdPl7FgU/Mrko='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'client': {'sha1': 'distro'}, 'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3'}}, {'ceph': {'create_rbd_pool': False, 'cluster-conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 2, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mon': {'mon op complaint time': 120, 'mon warn on pool no app': False}, 'osd': {'osd op complaint time': 180}}, 'conf': {'client': {'pid file': ''}, 'global': {'ms die on skipped message': False}, 'mgr': {'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'bad backtrace', 'object missing on disk', 'error reading table object', 'error reading sessionmap', 'unmatched fragstat', 'unmatched rstat', 'was unreadable, recreating it now', 'Scrub error on inode', 'Metadata damage detected', 'MDS_FAILED', 'MDS_DAMAGE'], 'sha1': '488c4d2c8097ced3f3bab85cfc6687562b3affe3', 'cluster': 'ceph'}}, {'sequential': ['mgrmodules']}, {'kclient': None}, {'cephfs_test_runner': {'modules': ['tasks.cephfs.test_recovery_pool']}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': ["local function is_kupstream() return yaml.ktype == 'upstream' end", "local function is_kdistro() return yaml.ktype == 'distro' end", 'local function is_fuse() return false end', 'local function syntax_version() return yaml.overrides.kclient.syntax end']}, 'teuthology_branch': 'main', 'teuthology_sha1': '43b88056bcf6a0548bdd2d0502e56290cbae52df', 'timestamp': '2024-08-14_02:13:41', 'tube': 'smithi', 'user': 'pdonnell', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-08-14T06:51:33.681 DEBUG:teuthology.orchestra.run.smithi098:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2024-08-14T06:51:33.684 DEBUG:teuthology.orchestra.run.smithi191:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2024-08-14T06:51:33.730 DEBUG:teuthology.orchestra.run.smithi098:> sudo install -d -m0777 -- /var/run/ceph 2024-08-14T06:51:33.739 DEBUG:teuthology.orchestra.run.smithi191:> sudo install -d -m0777 -- /var/run/ceph 2024-08-14T06:51:33.796 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:33.796 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/scratch_devs of=/dev/stdout 2024-08-14T06:51:33.817 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-14T06:51:33.817 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_1 2024-08-14T06:51:33.874 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-14T06:51:33.874 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-14T06:51:33.874 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 5h/5d Inode: 676 Links: 1 2024-08-14T06:51:33.874 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-14T06:51:33.874 INFO:teuthology.orchestra.run.smithi098.stdout:Context: system_u:object_r:device_t:s0 2024-08-14T06:51:33.874 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2024-08-14 06:51:22.690758919 +0000 2024-08-14T06:51:33.874 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2024-08-14 06:48:55.973360479 +0000 2024-08-14T06:51:33.874 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2024-08-14 06:48:55.973360479 +0000 2024-08-14T06:51:33.874 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: 2024-08-14 06:48:55.973360479 +0000 2024-08-14T06:51:33.875 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-14T06:51:33.937 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2024-08-14T06:51:33.937 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2024-08-14T06:51:33.937 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000196963 s, 2.6 MB/s 2024-08-14T06:51:33.938 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-14T06:51:33.995 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_2 2024-08-14T06:51:34.050 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-14T06:51:34.051 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-14T06:51:34.051 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 5h/5d Inode: 695 Links: 1 2024-08-14T06:51:34.051 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-14T06:51:34.051 INFO:teuthology.orchestra.run.smithi098.stdout:Context: system_u:object_r:device_t:s0 2024-08-14T06:51:34.051 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2024-08-14 06:51:22.690758919 +0000 2024-08-14T06:51:34.051 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2024-08-14 06:48:56.312355670 +0000 2024-08-14T06:51:34.051 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2024-08-14 06:48:56.312355670 +0000 2024-08-14T06:51:34.051 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: 2024-08-14 06:48:56.312355670 +0000 2024-08-14T06:51:34.051 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-14T06:51:34.113 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2024-08-14T06:51:34.113 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2024-08-14T06:51:34.114 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000237967 s, 2.2 MB/s 2024-08-14T06:51:34.115 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-14T06:51:34.172 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_3 2024-08-14T06:51:34.227 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-14T06:51:34.228 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-14T06:51:34.228 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 5h/5d Inode: 714 Links: 1 2024-08-14T06:51:34.228 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-14T06:51:34.228 INFO:teuthology.orchestra.run.smithi098.stdout:Context: system_u:object_r:device_t:s0 2024-08-14T06:51:34.228 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2024-08-14 06:51:22.691758905 +0000 2024-08-14T06:51:34.228 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2024-08-14 06:48:56.647350918 +0000 2024-08-14T06:51:34.228 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2024-08-14 06:48:56.647350918 +0000 2024-08-14T06:51:34.228 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: 2024-08-14 06:48:56.647350918 +0000 2024-08-14T06:51:34.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-14T06:51:34.291 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2024-08-14T06:51:34.291 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2024-08-14T06:51:34.291 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000181063 s, 2.8 MB/s 2024-08-14T06:51:34.292 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-14T06:51:34.349 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_4 2024-08-14T06:51:34.403 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-14T06:51:34.404 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-14T06:51:34.404 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 5h/5d Inode: 733 Links: 1 2024-08-14T06:51:34.404 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-14T06:51:34.404 INFO:teuthology.orchestra.run.smithi098.stdout:Context: system_u:object_r:device_t:s0 2024-08-14T06:51:34.404 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2024-08-14 06:51:22.691758905 +0000 2024-08-14T06:51:34.404 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2024-08-14 06:48:56.999345924 +0000 2024-08-14T06:51:34.404 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2024-08-14 06:48:56.999345924 +0000 2024-08-14T06:51:34.404 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: 2024-08-14 06:48:56.999345924 +0000 2024-08-14T06:51:34.404 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-14T06:51:34.466 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2024-08-14T06:51:34.466 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2024-08-14T06:51:34.466 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000192789 s, 2.7 MB/s 2024-08-14T06:51:34.467 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-14T06:51:34.524 INFO:tasks.ceph:osd dev map: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-08-14T06:51:34.524 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:51:34.524 DEBUG:teuthology.orchestra.run.smithi191:> dd if=/scratch_devs of=/dev/stdout 2024-08-14T06:51:34.542 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-14T06:51:34.542 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_1 2024-08-14T06:51:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-14T06:51:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-14T06:51:34.600 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 5h/5d Inode: 673 Links: 1 2024-08-14T06:51:34.600 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-14T06:51:34.600 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2024-08-14T06:51:34.600 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2024-08-14 06:51:07.902831608 +0000 2024-08-14T06:51:34.600 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2024-08-14 06:48:58.829258880 +0000 2024-08-14T06:51:34.600 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2024-08-14 06:48:58.829258880 +0000 2024-08-14T06:51:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: 2024-08-14 06:48:58.829258880 +0000 2024-08-14T06:51:34.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-14T06:51:34.665 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2024-08-14T06:51:34.665 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2024-08-14T06:51:34.665 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000269449 s, 1.9 MB/s 2024-08-14T06:51:34.667 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-14T06:51:34.725 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_2 2024-08-14T06:51:34.782 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-14T06:51:34.782 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-14T06:51:34.782 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 5h/5d Inode: 694 Links: 1 2024-08-14T06:51:34.782 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-14T06:51:34.782 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2024-08-14T06:51:34.782 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2024-08-14 06:51:07.902831608 +0000 2024-08-14T06:51:34.783 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2024-08-14 06:48:59.166253053 +0000 2024-08-14T06:51:34.783 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2024-08-14 06:48:59.166253053 +0000 2024-08-14T06:51:34.783 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: 2024-08-14 06:48:59.166253053 +0000 2024-08-14T06:51:34.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-14T06:51:34.847 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2024-08-14T06:51:34.847 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2024-08-14T06:51:34.847 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000186465 s, 2.7 MB/s 2024-08-14T06:51:34.848 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-14T06:51:34.905 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_3 2024-08-14T06:51:34.962 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-14T06:51:34.962 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-14T06:51:34.962 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 5h/5d Inode: 711 Links: 1 2024-08-14T06:51:34.962 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-14T06:51:34.962 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2024-08-14T06:51:34.962 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2024-08-14 06:51:07.902831608 +0000 2024-08-14T06:51:34.962 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2024-08-14 06:48:59.495247366 +0000 2024-08-14T06:51:34.962 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2024-08-14 06:48:59.495247366 +0000 2024-08-14T06:51:34.962 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: 2024-08-14 06:48:59.495247366 +0000 2024-08-14T06:51:34.963 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-14T06:51:35.025 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2024-08-14T06:51:35.026 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2024-08-14T06:51:35.026 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000161718 s, 3.2 MB/s 2024-08-14T06:51:35.027 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-14T06:51:35.084 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_4 2024-08-14T06:51:35.140 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-14T06:51:35.141 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-14T06:51:35.141 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 5h/5d Inode: 732 Links: 1 2024-08-14T06:51:35.141 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-14T06:51:35.141 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2024-08-14T06:51:35.141 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2024-08-14 06:51:07.903831591 +0000 2024-08-14T06:51:35.141 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2024-08-14 06:48:59.825241661 +0000 2024-08-14T06:51:35.141 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2024-08-14 06:48:59.825241661 +0000 2024-08-14T06:51:35.141 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: 2024-08-14 06:48:59.825241661 +0000 2024-08-14T06:51:35.141 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-14T06:51:35.204 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2024-08-14T06:51:35.204 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2024-08-14T06:51:35.204 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000188177 s, 2.7 MB/s 2024-08-14T06:51:35.205 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-14T06:51:35.261 INFO:tasks.ceph:osd dev map: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-08-14T06:51:35.262 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi098.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi191.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2024-08-14T06:51:35.262 INFO:tasks.ceph:Generating config... 2024-08-14T06:51:35.263 INFO:tasks.ceph:[client] pid file = 2024-08-14T06:51:35.263 INFO:tasks.ceph:[global] ms die on skipped message = False 2024-08-14T06:51:35.263 INFO:tasks.ceph:[mgr] debug client = 20 2024-08-14T06:51:35.264 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-08-14T06:51:35.264 INFO:tasks.ceph:[mgr] debug ms = 1 2024-08-14T06:51:35.264 INFO:tasks.ceph:[mon] debug mon = 20 2024-08-14T06:51:35.264 INFO:tasks.ceph:[mon] debug ms = 1 2024-08-14T06:51:35.264 INFO:tasks.ceph:[mon] debug paxos = 20 2024-08-14T06:51:35.264 INFO:tasks.ceph:[mon] mon down mkfs grace = 300 2024-08-14T06:51:35.264 INFO:tasks.ceph:[osd] bdev async discard = True 2024-08-14T06:51:35.264 INFO:tasks.ceph:[osd] bdev enable discard = True 2024-08-14T06:51:35.264 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2024-08-14T06:51:35.264 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-08-14T06:51:35.264 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-08-14T06:51:35.264 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-08-14T06:51:35.264 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-08-14T06:51:35.264 INFO:tasks.ceph:[osd] debug ms = 1 2024-08-14T06:51:35.264 INFO:tasks.ceph:[osd] debug osd = 20 2024-08-14T06:51:35.265 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-08-14T06:51:35.265 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-08-14T06:51:35.265 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-08-14T06:51:35.265 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-08-14T06:51:35.265 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-08-14T06:51:35.265 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-08-14T06:51:35.265 INFO:tasks.ceph:Setting up mon.a... 2024-08-14T06:51:35.265 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring 2024-08-14T06:51:35.356 INFO:teuthology.orchestra.run.smithi098.stdout:creating /etc/ceph/ceph.keyring 2024-08-14T06:51:35.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring 2024-08-14T06:51:35.413 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-08-14T06:51:35.438 DEBUG:tasks.ceph:Ceph mon addresses: [('mon.a', '172.21.15.98'), ('mon.b', '172.21.15.191'), ('mon.c', '[v2:172.21.15.191:3301,v1:172.21.15.191:6790]')] 2024-08-14T06:51:35.438 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.98,172.21.15.191,[v2:172.21.15.191:3301,v1:172.21.15.191:6790]', 'ms die on skipped message': False}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false', 'debug client': 20}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false', 'mon down mkfs grace': 300}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'pid file': ''}, 'mon.a': {}, 'mon.b': {}, 'mon.c': {}, 'mds.a': {}, 'mds.c': {}, 'mds.b': {}, 'mds.d': {}} 2024-08-14T06:51:35.439 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:35.439 DEBUG:teuthology.orchestra.run.smithi098:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-08-14T06:51:35.493 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.98 --add b 172.21.15.191 --addv c '[v2:172.21.15.191:3301,v1:172.21.15.191:6790]' --print /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:51:35.590 INFO:teuthology.orchestra.run.smithi098.stdout:monmaptool: monmap file /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:51:35.590 INFO:teuthology.orchestra.run.smithi098.stdout:monmaptool: generated fsid 1a641e76-d282-43fb-b02c-28f3852c1ab1 2024-08-14T06:51:35.590 INFO:teuthology.orchestra.run.smithi098.stdout:setting min_mon_release = quincy 2024-08-14T06:51:35.590 INFO:teuthology.orchestra.run.smithi098.stdout:epoch 0 2024-08-14T06:51:35.591 INFO:teuthology.orchestra.run.smithi098.stdout:fsid 1a641e76-d282-43fb-b02c-28f3852c1ab1 2024-08-14T06:51:35.591 INFO:teuthology.orchestra.run.smithi098.stdout:last_changed 2024-08-14T06:51:35.590680+0000 2024-08-14T06:51:35.591 INFO:teuthology.orchestra.run.smithi098.stdout:created 2024-08-14T06:51:35.590680+0000 2024-08-14T06:51:35.591 INFO:teuthology.orchestra.run.smithi098.stdout:min_mon_release 17 (quincy) 2024-08-14T06:51:35.591 INFO:teuthology.orchestra.run.smithi098.stdout:election_strategy: 1 2024-08-14T06:51:35.591 INFO:teuthology.orchestra.run.smithi098.stdout:0: [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] mon.a 2024-08-14T06:51:35.591 INFO:teuthology.orchestra.run.smithi098.stdout:1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2024-08-14T06:51:35.591 INFO:teuthology.orchestra.run.smithi098.stdout:2: [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] mon.c 2024-08-14T06:51:35.591 INFO:teuthology.orchestra.run.smithi098.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/ceph.monmap (3 monitors) 2024-08-14T06:51:35.592 DEBUG:teuthology.orchestra.run.smithi098:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-08-14T06:51:35.607 INFO:tasks.ceph:Writing /etc/ceph/ceph.conf for FSID 1a641e76-d282-43fb-b02c-28f3852c1ab1... 2024-08-14T06:51:35.608 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-08-14T06:51:35.650 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-08-14T06:51:35.690 INFO:teuthology.orchestra.run.smithi098.stdout:[global] 2024-08-14T06:51:35.690 INFO:teuthology.orchestra.run.smithi098.stdout: chdir = "" 2024-08-14T06:51:35.690 INFO:teuthology.orchestra.run.smithi098.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-08-14T06:51:35.690 INFO:teuthology.orchestra.run.smithi098.stdout: auth supported = cephx 2024-08-14T06:51:35.690 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: filestore xattr use omap = true 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: mon clock drift allowed = 1.000 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: osd crush chooseleaf type = 0 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: auth debug = true 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: ms die on old message = true 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: ms die on bug = true 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: mon max pg per osd = 10000 # >= luminous 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: mon pg warn max object skew = 0 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.691 INFO:teuthology.orchestra.run.smithi098.stdout: # disable pg_autoscaler by default for new pools 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: osd_pool_default_pg_autoscale_mode = off 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: osd pool default size = 2 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd allow primary affinity = true 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd allow pg remap = true 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: mon warn on legacy crush tunables = false 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: mon warn on crush straw calc version zero = false 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: mon warn on no sortbitwise = false 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: mon warn on osd down out interval zero = false 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: mon warn on too few osds = false 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: mon_warn_on_pool_no_redundancy = false 2024-08-14T06:51:35.692 INFO:teuthology.orchestra.run.smithi098.stdout: mon_allow_pool_size_one = true 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: osd default data pool replay window = 5 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: mon allow pool delete = true 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: mon cluster log file level = debug 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: debug asserts on shutdown = true 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: mon health detail to clog = false 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: mon host = "172.21.15.98,172.21.15.191,[v2:172.21.15.191:3301,v1:172.21.15.191:6790]" 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: ms die on skipped message = False 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: fsid = 1a641e76-d282-43fb-b02c-28f3852c1ab1 2024-08-14T06:51:35.693 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.694 INFO:teuthology.orchestra.run.smithi098.stdout:[osd] 2024-08-14T06:51:35.694 INFO:teuthology.orchestra.run.smithi098.stdout: osd journal size = 100 2024-08-14T06:51:35.694 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.694 INFO:teuthology.orchestra.run.smithi098.stdout: osd scrub load threshold = 5.0 2024-08-14T06:51:35.694 INFO:teuthology.orchestra.run.smithi098.stdout: osd scrub max interval = 600 2024-08-14T06:51:35.694 INFO:teuthology.orchestra.run.smithi098.stdout: osd mclock profile = high_recovery_ops 2024-08-14T06:51:35.694 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.694 INFO:teuthology.orchestra.run.smithi098.stdout: osd recover clone overlap = true 2024-08-14T06:51:35.694 INFO:teuthology.orchestra.run.smithi098.stdout: osd recovery max chunk = 1048576 2024-08-14T06:51:35.694 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug shutdown = true 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug op order = true 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug verify stray on activate = true 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug trim objects = true 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: osd open classes on start = true 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug pg log writeout = true 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: osd deep scrub update digest min age = 30 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: osd map max advance = 10 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: journal zero on create = true 2024-08-14T06:51:35.695 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: filestore ondisk finisher threads = 3 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: filestore apply finisher threads = 3 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: bdev debug aio = true 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug misdirected ops = true 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: bdev async discard = True 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: bdev enable discard = True 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: bluestore allocator = bitmap 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: bluestore block size = 96636764160 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: bluestore fsck on mount = True 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: debug bluefs = 1/20 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: debug bluestore = 1/20 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: debug ms = 1 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: debug osd = 20 2024-08-14T06:51:35.696 INFO:teuthology.orchestra.run.smithi098.stdout: debug rocksdb = 4/10 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd backfillfull_ratio = 0.85 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd full ratio = 0.9 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd nearfull ratio = 0.8 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: osd failsafe full ratio = 0.95 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: osd objectstore = bluestore 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout:[mgr] 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: debug ms = 1 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: debug mgr = 20 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: debug mon = 20 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: debug auth = 20 2024-08-14T06:51:35.697 INFO:teuthology.orchestra.run.smithi098.stdout: mon reweight min pgs per osd = 4 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: mon reweight min bytes per osd = 10 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: mgr/telemetry/nag = false 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: debug client = 20 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout:[mon] 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: debug ms = 1 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: debug mon = 20 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: debug paxos = 20 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: debug auth = 20 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: mon data avail warn = 5 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: mon mgr mkfs grace = 240 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: mon reweight min pgs per osd = 4 2024-08-14T06:51:35.698 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd reporter subtree level = osd 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd prime pg temp = true 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: mon reweight min bytes per osd = 10 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: auth mon ticket ttl = 660 # 11m 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: auth service ticket ttl = 240 # 4m 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: # don't complain about insecure global_id in the test suite 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: # 1m isn't quite enough 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: mon_down_mkfs_grace = 2m 2024-08-14T06:51:35.699 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout: mon_warn_on_filestore_osds = false 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout: mon down mkfs grace = 300 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout:[client] 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout: rgw cache enabled = true 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout: rgw enable ops log = true 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout: rgw enable usage log = true 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout: pid file = "" 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout:[mon.a] 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout:[mon.b] 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout:[mon.c] 2024-08-14T06:51:35.700 INFO:teuthology.orchestra.run.smithi098.stdout:[mds.a] 2024-08-14T06:51:35.701 INFO:teuthology.orchestra.run.smithi098.stdout:[mds.c] 2024-08-14T06:51:35.701 INFO:teuthology.orchestra.run.smithi098.stdout:[mds.b] 2024-08-14T06:51:35.701 INFO:teuthology.orchestra.run.smithi098.stdout:[mds.d] 2024-08-14T06:51:35.701 INFO:teuthology.orchestra.run.smithi191.stdout:[global] 2024-08-14T06:51:35.701 INFO:teuthology.orchestra.run.smithi191.stdout: chdir = "" 2024-08-14T06:51:35.701 INFO:teuthology.orchestra.run.smithi191.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: auth supported = cephx 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: filestore xattr use omap = true 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: mon clock drift allowed = 1.000 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: osd crush chooseleaf type = 0 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: auth debug = true 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: ms die on old message = true 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: ms die on bug = true 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: mon max pg per osd = 10000 # >= luminous 2024-08-14T06:51:35.702 INFO:teuthology.orchestra.run.smithi191.stdout: mon pg warn max object skew = 0 2024-08-14T06:51:35.703 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.703 INFO:teuthology.orchestra.run.smithi191.stdout: # disable pg_autoscaler by default for new pools 2024-08-14T06:51:35.703 INFO:teuthology.orchestra.run.smithi191.stdout: osd_pool_default_pg_autoscale_mode = off 2024-08-14T06:51:35.703 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.703 INFO:teuthology.orchestra.run.smithi191.stdout: osd pool default size = 2 2024-08-14T06:51:35.703 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.703 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd allow primary affinity = true 2024-08-14T06:51:35.703 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd allow pg remap = true 2024-08-14T06:51:35.703 INFO:teuthology.orchestra.run.smithi191.stdout: mon warn on legacy crush tunables = false 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: mon warn on crush straw calc version zero = false 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: mon warn on no sortbitwise = false 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: mon warn on osd down out interval zero = false 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: mon warn on too few osds = false 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: mon_warn_on_pool_no_redundancy = false 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: mon_allow_pool_size_one = true 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: osd default data pool replay window = 5 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: mon allow pool delete = true 2024-08-14T06:51:35.704 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: mon cluster log file level = debug 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: debug asserts on shutdown = true 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: mon health detail to clog = false 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: mon host = "172.21.15.98,172.21.15.191,[v2:172.21.15.191:3301,v1:172.21.15.191:6790]" 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: ms die on skipped message = False 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: fsid = 1a641e76-d282-43fb-b02c-28f3852c1ab1 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout:[osd] 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: osd journal size = 100 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: osd scrub load threshold = 5.0 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: osd scrub max interval = 600 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: osd mclock profile = high_recovery_ops 2024-08-14T06:51:35.705 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: osd recover clone overlap = true 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: osd recovery max chunk = 1048576 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug shutdown = true 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug op order = true 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug verify stray on activate = true 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug trim objects = true 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: osd open classes on start = true 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug pg log writeout = true 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: osd deep scrub update digest min age = 30 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.706 INFO:teuthology.orchestra.run.smithi191.stdout: osd map max advance = 10 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: journal zero on create = true 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: filestore ondisk finisher threads = 3 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: filestore apply finisher threads = 3 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: bdev debug aio = true 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug misdirected ops = true 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: bdev async discard = True 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: bdev enable discard = True 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: bluestore allocator = bitmap 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: bluestore block size = 96636764160 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: bluestore fsck on mount = True 2024-08-14T06:51:35.707 INFO:teuthology.orchestra.run.smithi191.stdout: debug bluefs = 1/20 2024-08-14T06:51:35.708 INFO:teuthology.orchestra.run.smithi191.stdout: debug bluestore = 1/20 2024-08-14T06:51:35.708 INFO:teuthology.orchestra.run.smithi191.stdout: debug ms = 1 2024-08-14T06:51:35.708 INFO:teuthology.orchestra.run.smithi191.stdout: debug osd = 20 2024-08-14T06:51:35.708 INFO:teuthology.orchestra.run.smithi191.stdout: debug rocksdb = 4/10 2024-08-14T06:51:35.708 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd backfillfull_ratio = 0.85 2024-08-14T06:51:35.708 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd full ratio = 0.9 2024-08-14T06:51:35.708 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd nearfull ratio = 0.8 2024-08-14T06:51:35.708 INFO:teuthology.orchestra.run.smithi191.stdout: osd failsafe full ratio = 0.95 2024-08-14T06:51:35.708 INFO:teuthology.orchestra.run.smithi191.stdout: osd objectstore = bluestore 2024-08-14T06:51:35.708 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.709 INFO:teuthology.orchestra.run.smithi191.stdout:[mgr] 2024-08-14T06:51:35.709 INFO:teuthology.orchestra.run.smithi191.stdout: debug ms = 1 2024-08-14T06:51:35.709 INFO:teuthology.orchestra.run.smithi191.stdout: debug mgr = 20 2024-08-14T06:51:35.709 INFO:teuthology.orchestra.run.smithi191.stdout: debug mon = 20 2024-08-14T06:51:35.709 INFO:teuthology.orchestra.run.smithi191.stdout: debug auth = 20 2024-08-14T06:51:35.710 INFO:teuthology.orchestra.run.smithi191.stdout: mon reweight min pgs per osd = 4 2024-08-14T06:51:35.710 INFO:teuthology.orchestra.run.smithi191.stdout: mon reweight min bytes per osd = 10 2024-08-14T06:51:35.710 INFO:teuthology.orchestra.run.smithi191.stdout: mgr/telemetry/nag = false 2024-08-14T06:51:35.710 INFO:teuthology.orchestra.run.smithi191.stdout: debug client = 20 2024-08-14T06:51:35.711 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.711 INFO:teuthology.orchestra.run.smithi191.stdout:[mon] 2024-08-14T06:51:35.711 INFO:teuthology.orchestra.run.smithi191.stdout: debug ms = 1 2024-08-14T06:51:35.711 INFO:teuthology.orchestra.run.smithi191.stdout: debug mon = 20 2024-08-14T06:51:35.711 INFO:teuthology.orchestra.run.smithi191.stdout: debug paxos = 20 2024-08-14T06:51:35.711 INFO:teuthology.orchestra.run.smithi191.stdout: debug auth = 20 2024-08-14T06:51:35.711 INFO:teuthology.orchestra.run.smithi191.stdout: mon data avail warn = 5 2024-08-14T06:51:35.711 INFO:teuthology.orchestra.run.smithi191.stdout: mon mgr mkfs grace = 240 2024-08-14T06:51:35.711 INFO:teuthology.orchestra.run.smithi191.stdout: mon reweight min pgs per osd = 4 2024-08-14T06:51:35.711 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd reporter subtree level = osd 2024-08-14T06:51:35.712 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd prime pg temp = true 2024-08-14T06:51:35.712 INFO:teuthology.orchestra.run.smithi191.stdout: mon reweight min bytes per osd = 10 2024-08-14T06:51:35.712 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.712 INFO:teuthology.orchestra.run.smithi191.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-08-14T06:51:35.712 INFO:teuthology.orchestra.run.smithi191.stdout: auth mon ticket ttl = 660 # 11m 2024-08-14T06:51:35.712 INFO:teuthology.orchestra.run.smithi191.stdout: auth service ticket ttl = 240 # 4m 2024-08-14T06:51:35.712 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.712 INFO:teuthology.orchestra.run.smithi191.stdout: # don't complain about insecure global_id in the test suite 2024-08-14T06:51:35.712 INFO:teuthology.orchestra.run.smithi191.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-08-14T06:51:35.712 INFO:teuthology.orchestra.run.smithi191.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-08-14T06:51:35.713 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.713 INFO:teuthology.orchestra.run.smithi191.stdout: # 1m isn't quite enough 2024-08-14T06:51:35.713 INFO:teuthology.orchestra.run.smithi191.stdout: mon_down_mkfs_grace = 2m 2024-08-14T06:51:35.713 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.713 INFO:teuthology.orchestra.run.smithi191.stdout: mon_warn_on_filestore_osds = false 2024-08-14T06:51:35.713 INFO:teuthology.orchestra.run.smithi191.stdout: mon down mkfs grace = 300 2024-08-14T06:51:35.713 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-08-14T06:51:35.713 INFO:teuthology.orchestra.run.smithi191.stdout:[client] 2024-08-14T06:51:35.713 INFO:teuthology.orchestra.run.smithi191.stdout: rgw cache enabled = true 2024-08-14T06:51:35.714 INFO:teuthology.orchestra.run.smithi191.stdout: rgw enable ops log = true 2024-08-14T06:51:35.714 INFO:teuthology.orchestra.run.smithi191.stdout: rgw enable usage log = true 2024-08-14T06:51:35.714 INFO:teuthology.orchestra.run.smithi191.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-08-14T06:51:35.714 INFO:teuthology.orchestra.run.smithi191.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-08-14T06:51:35.714 INFO:teuthology.orchestra.run.smithi191.stdout: pid file = "" 2024-08-14T06:51:35.715 INFO:teuthology.orchestra.run.smithi191.stdout:[mon.a] 2024-08-14T06:51:35.715 INFO:teuthology.orchestra.run.smithi191.stdout:[mon.b] 2024-08-14T06:51:35.715 INFO:teuthology.orchestra.run.smithi191.stdout:[mon.c] 2024-08-14T06:51:35.715 INFO:teuthology.orchestra.run.smithi191.stdout:[mds.a] 2024-08-14T06:51:35.715 INFO:teuthology.orchestra.run.smithi191.stdout:[mds.c] 2024-08-14T06:51:35.715 INFO:teuthology.orchestra.run.smithi191.stdout:[mds.b] 2024-08-14T06:51:35.716 INFO:teuthology.orchestra.run.smithi191.stdout:[mds.d] 2024-08-14T06:51:35.718 INFO:tasks.ceph:Creating admin key on mon.a... 2024-08-14T06:51:35.719 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/ceph.keyring 2024-08-14T06:51:35.823 INFO:tasks.ceph:Copying monmap to all nodes... 2024-08-14T06:51:35.823 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:35.823 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/etc/ceph/ceph.keyring of=/dev/stdout 2024-08-14T06:51:35.839 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:35.839 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/home/ubuntu/cephtest/ceph.monmap of=/dev/stdout 2024-08-14T06:51:35.894 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi098.front.sepia.ceph.com 2024-08-14T06:51:35.895 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:35.895 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/ceph/ceph.keyring 2024-08-14T06:51:35.895 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-08-14T06:51:35.969 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:51:35.969 DEBUG:teuthology.orchestra.run.smithi098:> dd of=/home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:51:36.024 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi191.front.sepia.ceph.com 2024-08-14T06:51:36.025 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:51:36.025 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/ceph/ceph.keyring 2024-08-14T06:51:36.025 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-08-14T06:51:36.060 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:51:36.060 DEBUG:teuthology.orchestra.run.smithi191:> dd of=/home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:51:36.116 INFO:tasks.ceph:Setting up mon nodes... 2024-08-14T06:51:36.116 INFO:tasks.ceph:Setting up mgr nodes... 2024-08-14T06:51:36.117 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/mgr/ceph-y && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.y /var/lib/ceph/mgr/ceph-y/keyring 2024-08-14T06:51:36.177 INFO:teuthology.orchestra.run.smithi098.stdout:creating /var/lib/ceph/mgr/ceph-y/keyring 2024-08-14T06:51:36.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/mgr/ceph-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/ceph-x/keyring 2024-08-14T06:51:36.253 INFO:teuthology.orchestra.run.smithi191.stdout:creating /var/lib/ceph/mgr/ceph-x/keyring 2024-08-14T06:51:36.255 INFO:tasks.ceph:Setting up mds nodes... 2024-08-14T06:51:36.255 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/mds/ceph-a && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.a /var/lib/ceph/mds/ceph-a/keyring 2024-08-14T06:51:36.316 INFO:teuthology.orchestra.run.smithi098.stdout:creating /var/lib/ceph/mds/ceph-a/keyring 2024-08-14T06:51:36.319 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-a 2024-08-14T06:51:36.343 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/mds/ceph-c && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.c /var/lib/ceph/mds/ceph-c/keyring 2024-08-14T06:51:36.443 INFO:teuthology.orchestra.run.smithi098.stdout:creating /var/lib/ceph/mds/ceph-c/keyring 2024-08-14T06:51:36.446 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-c 2024-08-14T06:51:36.471 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/mds/ceph-b && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.b /var/lib/ceph/mds/ceph-b/keyring 2024-08-14T06:51:36.532 INFO:teuthology.orchestra.run.smithi191.stdout:creating /var/lib/ceph/mds/ceph-b/keyring 2024-08-14T06:51:36.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-b 2024-08-14T06:51:36.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/mds/ceph-d && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.d /var/lib/ceph/mds/ceph-d/keyring 2024-08-14T06:51:36.660 INFO:teuthology.orchestra.run.smithi191.stdout:creating /var/lib/ceph/mds/ceph-d/keyring 2024-08-14T06:51:36.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-d 2024-08-14T06:51:36.688 INFO:tasks.ceph_client:Setting up client nodes... 2024-08-14T06:51:36.688 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/ceph.client.0.keyring && sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-08-14T06:51:36.740 INFO:teuthology.orchestra.run.smithi098.stdout:creating /etc/ceph/ceph.client.0.keyring 2024-08-14T06:51:36.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.2 /etc/ceph/ceph.client.2.keyring && sudo chmod 0644 /etc/ceph/ceph.client.2.keyring 2024-08-14T06:51:36.804 INFO:teuthology.orchestra.run.smithi098.stdout:creating /etc/ceph/ceph.client.2.keyring 2024-08-14T06:51:36.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/ceph.client.1.keyring && sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-08-14T06:51:36.868 INFO:teuthology.orchestra.run.smithi191.stdout:creating /etc/ceph/ceph.client.1.keyring 2024-08-14T06:51:36.882 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.3 /etc/ceph/ceph.client.3.keyring && sudo chmod 0644 /etc/ceph/ceph.client.3.keyring 2024-08-14T06:51:36.933 INFO:teuthology.orchestra.run.smithi191.stdout:creating /etc/ceph/ceph.client.3.keyring 2024-08-14T06:51:36.946 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-08-14T06:51:36.946 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi098.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi191.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2024-08-14T06:51:36.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/osd/ceph-0 2024-08-14T06:51:36.971 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-08-14T06:51:36.971 INFO:tasks.ceph:role: osd.0 2024-08-14T06:51:36.971 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi098.front.sepia.ceph.com 2024-08-14T06:51:36.971 DEBUG:teuthology.orchestra.run.smithi098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-08-14T06:51:37.043 INFO:teuthology.orchestra.run.smithi098.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-08-14T06:51:37.044 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-14T06:51:37.044 INFO:teuthology.orchestra.run.smithi098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-14T06:51:37.044 INFO:teuthology.orchestra.run.smithi098.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-14T06:51:37.044 INFO:teuthology.orchestra.run.smithi098.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-14T06:51:37.044 INFO:teuthology.orchestra.run.smithi098.stdout: = sunit=0 swidth=0 blks 2024-08-14T06:51:37.044 INFO:teuthology.orchestra.run.smithi098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-14T06:51:37.044 INFO:teuthology.orchestra.run.smithi098.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-14T06:51:37.044 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-14T06:51:37.044 INFO:teuthology.orchestra.run.smithi098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-14T06:51:37.557 INFO:teuthology.orchestra.run.smithi098.stdout:Discarding blocks...Done. 2024-08-14T06:51:37.558 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi098.front.sepia.ceph.com -o noatime 2024-08-14T06:51:37.559 DEBUG:teuthology.orchestra.run.smithi098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-0 2024-08-14T06:51:37.685 DEBUG:teuthology.orchestra.run.smithi098:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-0 2024-08-14T06:51:37.717 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/osd/ceph-1 2024-08-14T06:51:37.782 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-08-14T06:51:37.783 INFO:tasks.ceph:role: osd.1 2024-08-14T06:51:37.783 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi098.front.sepia.ceph.com 2024-08-14T06:51:37.783 DEBUG:teuthology.orchestra.run.smithi098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-08-14T06:51:37.854 INFO:teuthology.orchestra.run.smithi098.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-08-14T06:51:37.854 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-14T06:51:37.854 INFO:teuthology.orchestra.run.smithi098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-14T06:51:37.855 INFO:teuthology.orchestra.run.smithi098.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-14T06:51:37.855 INFO:teuthology.orchestra.run.smithi098.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-14T06:51:37.855 INFO:teuthology.orchestra.run.smithi098.stdout: = sunit=0 swidth=0 blks 2024-08-14T06:51:37.855 INFO:teuthology.orchestra.run.smithi098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-14T06:51:37.855 INFO:teuthology.orchestra.run.smithi098.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-14T06:51:37.855 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-14T06:51:37.855 INFO:teuthology.orchestra.run.smithi098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-14T06:51:38.308 INFO:teuthology.orchestra.run.smithi098.stdout:Discarding blocks...Done. 2024-08-14T06:51:38.310 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi098.front.sepia.ceph.com -o noatime 2024-08-14T06:51:38.310 DEBUG:teuthology.orchestra.run.smithi098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-1 2024-08-14T06:51:38.394 DEBUG:teuthology.orchestra.run.smithi098:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-1 2024-08-14T06:51:38.464 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/osd/ceph-2 2024-08-14T06:51:38.530 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-08-14T06:51:38.530 INFO:tasks.ceph:role: osd.2 2024-08-14T06:51:38.530 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi098.front.sepia.ceph.com 2024-08-14T06:51:38.530 DEBUG:teuthology.orchestra.run.smithi098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-08-14T06:51:38.602 INFO:teuthology.orchestra.run.smithi098.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-08-14T06:51:38.602 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-14T06:51:38.602 INFO:teuthology.orchestra.run.smithi098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-14T06:51:38.603 INFO:teuthology.orchestra.run.smithi098.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-14T06:51:38.603 INFO:teuthology.orchestra.run.smithi098.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-14T06:51:38.603 INFO:teuthology.orchestra.run.smithi098.stdout: = sunit=0 swidth=0 blks 2024-08-14T06:51:38.603 INFO:teuthology.orchestra.run.smithi098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-14T06:51:38.603 INFO:teuthology.orchestra.run.smithi098.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-14T06:51:38.603 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-14T06:51:38.603 INFO:teuthology.orchestra.run.smithi098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-14T06:51:39.089 INFO:teuthology.orchestra.run.smithi098.stdout:Discarding blocks...Done. 2024-08-14T06:51:39.091 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi098.front.sepia.ceph.com -o noatime 2024-08-14T06:51:39.092 DEBUG:teuthology.orchestra.run.smithi098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-2 2024-08-14T06:51:39.224 DEBUG:teuthology.orchestra.run.smithi098:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-2 2024-08-14T06:51:39.253 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/osd/ceph-3 2024-08-14T06:51:39.318 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-08-14T06:51:39.318 INFO:tasks.ceph:role: osd.3 2024-08-14T06:51:39.318 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi098.front.sepia.ceph.com 2024-08-14T06:51:39.318 DEBUG:teuthology.orchestra.run.smithi098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2024-08-14T06:51:39.390 INFO:teuthology.orchestra.run.smithi098.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2024-08-14T06:51:39.538 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-14T06:51:39.538 INFO:teuthology.orchestra.run.smithi098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-14T06:51:39.538 INFO:teuthology.orchestra.run.smithi098.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-14T06:51:39.538 INFO:teuthology.orchestra.run.smithi098.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-14T06:51:39.539 INFO:teuthology.orchestra.run.smithi098.stdout: = sunit=0 swidth=0 blks 2024-08-14T06:51:39.539 INFO:teuthology.orchestra.run.smithi098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-14T06:51:39.539 INFO:teuthology.orchestra.run.smithi098.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-14T06:51:39.539 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-14T06:51:39.539 INFO:teuthology.orchestra.run.smithi098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-14T06:51:39.924 INFO:teuthology.orchestra.run.smithi098.stdout:Discarding blocks...Done. 2024-08-14T06:51:39.926 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi098.front.sepia.ceph.com -o noatime 2024-08-14T06:51:39.926 DEBUG:teuthology.orchestra.run.smithi098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-3 2024-08-14T06:51:40.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-3 2024-08-14T06:51:40.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:51:40.205 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:40.203+0000 7f7b78a0f640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2024-08-14T06:51:40.205 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:40.204+0000 7f7b78a0f640 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2024-08-14T06:51:40.205 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:40.204+0000 7f7b78a0f640 -1 bdev(0x560f5d3ab900 /var/lib/ceph/osd/ceph-0/block) open stat got: (1) Operation not permitted 2024-08-14T06:51:40.205 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:40.204+0000 7f7b78a0f640 -1 bluestore(/var/lib/ceph/osd/ceph-0) _read_fsid unparsable uuid 2024-08-14T06:51:43.625 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-14T06:51:43.654 DEBUG:teuthology.orchestra.run.smithi098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:51:43.756 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:43.755+0000 7f1497c1b640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2024-08-14T06:51:43.756 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:43.755+0000 7f1497c1b640 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2024-08-14T06:51:43.756 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:43.755+0000 7f1497c1b640 -1 bdev(0x563855ce5900 /var/lib/ceph/osd/ceph-1/block) open stat got: (1) Operation not permitted 2024-08-14T06:51:43.756 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:43.755+0000 7f1497c1b640 -1 bluestore(/var/lib/ceph/osd/ceph-1) _read_fsid unparsable uuid 2024-08-14T06:51:47.174 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-14T06:51:47.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:51:47.305 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:47.304+0000 7f7c62c0d640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2024-08-14T06:51:47.305 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:47.304+0000 7f7c62c0d640 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2024-08-14T06:51:47.305 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:47.305+0000 7f7c62c0d640 -1 bdev(0x55dfd62af900 /var/lib/ceph/osd/ceph-2/block) open stat got: (1) Operation not permitted 2024-08-14T06:51:47.306 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:47.305+0000 7f7c62c0d640 -1 bluestore(/var/lib/ceph/osd/ceph-2) _read_fsid unparsable uuid 2024-08-14T06:51:50.719 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-14T06:51:50.748 DEBUG:teuthology.orchestra.run.smithi098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:51:50.850 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:50.849+0000 7ff1e2215640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2024-08-14T06:51:50.850 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:50.849+0000 7ff1e2215640 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2024-08-14T06:51:50.850 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:50.849+0000 7ff1e2215640 -1 bdev(0x55584656b900 /var/lib/ceph/osd/ceph-3/block) open stat got: (1) Operation not permitted 2024-08-14T06:51:50.850 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:51:50.849+0000 7ff1e2215640 -1 bluestore(/var/lib/ceph/osd/ceph-3) _read_fsid unparsable uuid 2024-08-14T06:51:54.254 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-14T06:51:54.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/osd/ceph-4 2024-08-14T06:51:54.310 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-08-14T06:51:54.311 INFO:tasks.ceph:role: osd.4 2024-08-14T06:51:54.311 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi191.front.sepia.ceph.com 2024-08-14T06:51:54.311 DEBUG:teuthology.orchestra.run.smithi191:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-08-14T06:51:54.386 INFO:teuthology.orchestra.run.smithi191.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-08-14T06:51:54.387 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-14T06:51:54.387 INFO:teuthology.orchestra.run.smithi191.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-14T06:51:54.387 INFO:teuthology.orchestra.run.smithi191.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-14T06:51:54.387 INFO:teuthology.orchestra.run.smithi191.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-14T06:51:54.387 INFO:teuthology.orchestra.run.smithi191.stdout: = sunit=0 swidth=0 blks 2024-08-14T06:51:54.387 INFO:teuthology.orchestra.run.smithi191.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-14T06:51:54.387 INFO:teuthology.orchestra.run.smithi191.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-14T06:51:54.387 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-14T06:51:54.387 INFO:teuthology.orchestra.run.smithi191.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-14T06:51:54.878 INFO:teuthology.orchestra.run.smithi191.stdout:Discarding blocks...Done. 2024-08-14T06:51:54.880 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi191.front.sepia.ceph.com -o noatime 2024-08-14T06:51:54.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-4 2024-08-14T06:51:55.022 DEBUG:teuthology.orchestra.run.smithi191:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-4 2024-08-14T06:51:55.055 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/osd/ceph-5 2024-08-14T06:51:55.121 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-08-14T06:51:55.121 INFO:tasks.ceph:role: osd.5 2024-08-14T06:51:55.122 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi191.front.sepia.ceph.com 2024-08-14T06:51:55.122 DEBUG:teuthology.orchestra.run.smithi191:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-08-14T06:51:55.196 INFO:teuthology.orchestra.run.smithi191.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-08-14T06:51:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-14T06:51:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-14T06:51:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-14T06:51:55.197 INFO:teuthology.orchestra.run.smithi191.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-14T06:51:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: = sunit=0 swidth=0 blks 2024-08-14T06:51:55.197 INFO:teuthology.orchestra.run.smithi191.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-14T06:51:55.197 INFO:teuthology.orchestra.run.smithi191.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-14T06:51:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-14T06:51:55.198 INFO:teuthology.orchestra.run.smithi191.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-14T06:51:55.639 INFO:teuthology.orchestra.run.smithi191.stdout:Discarding blocks...Done. 2024-08-14T06:51:55.640 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi191.front.sepia.ceph.com -o noatime 2024-08-14T06:51:55.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-5 2024-08-14T06:51:55.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-5 2024-08-14T06:51:55.792 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/osd/ceph-6 2024-08-14T06:51:55.857 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-08-14T06:51:55.858 INFO:tasks.ceph:role: osd.6 2024-08-14T06:51:55.858 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi191.front.sepia.ceph.com 2024-08-14T06:51:55.858 DEBUG:teuthology.orchestra.run.smithi191:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-08-14T06:51:55.929 INFO:teuthology.orchestra.run.smithi191.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-08-14T06:51:55.929 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-14T06:51:55.929 INFO:teuthology.orchestra.run.smithi191.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-14T06:51:55.929 INFO:teuthology.orchestra.run.smithi191.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-14T06:51:55.930 INFO:teuthology.orchestra.run.smithi191.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-14T06:51:55.930 INFO:teuthology.orchestra.run.smithi191.stdout: = sunit=0 swidth=0 blks 2024-08-14T06:51:55.930 INFO:teuthology.orchestra.run.smithi191.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-14T06:51:55.930 INFO:teuthology.orchestra.run.smithi191.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-14T06:51:55.930 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-14T06:51:55.930 INFO:teuthology.orchestra.run.smithi191.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-14T06:51:56.417 INFO:teuthology.orchestra.run.smithi191.stdout:Discarding blocks...Done. 2024-08-14T06:51:56.419 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi191.front.sepia.ceph.com -o noatime 2024-08-14T06:51:56.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-6 2024-08-14T06:51:56.566 DEBUG:teuthology.orchestra.run.smithi191:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-6 2024-08-14T06:51:56.597 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/osd/ceph-7 2024-08-14T06:51:56.661 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-08-14T06:51:56.661 INFO:tasks.ceph:role: osd.7 2024-08-14T06:51:56.662 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi191.front.sepia.ceph.com 2024-08-14T06:51:56.662 DEBUG:teuthology.orchestra.run.smithi191:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2024-08-14T06:51:56.734 INFO:teuthology.orchestra.run.smithi191.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2024-08-14T06:51:56.734 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-14T06:51:56.734 INFO:teuthology.orchestra.run.smithi191.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-14T06:51:56.734 INFO:teuthology.orchestra.run.smithi191.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-14T06:51:56.734 INFO:teuthology.orchestra.run.smithi191.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-14T06:51:56.734 INFO:teuthology.orchestra.run.smithi191.stdout: = sunit=0 swidth=0 blks 2024-08-14T06:51:56.734 INFO:teuthology.orchestra.run.smithi191.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-14T06:51:56.734 INFO:teuthology.orchestra.run.smithi191.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-14T06:51:56.734 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-14T06:51:56.734 INFO:teuthology.orchestra.run.smithi191.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-14T06:51:57.255 INFO:teuthology.orchestra.run.smithi191.stdout:Discarding blocks...Done. 2024-08-14T06:51:57.257 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi191.front.sepia.ceph.com -o noatime 2024-08-14T06:51:57.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-7 2024-08-14T06:51:57.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-7 2024-08-14T06:51:57.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 4 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:51:57.528 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:51:57.527+0000 7f7f9f6dd640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-4/keyring: can't open /var/lib/ceph/osd/ceph-4/keyring: (2) No such file or directory 2024-08-14T06:51:57.529 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:51:57.527+0000 7f7f9f6dd640 -1 created new key in keyring /var/lib/ceph/osd/ceph-4/keyring 2024-08-14T06:51:57.529 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:51:57.527+0000 7f7f9f6dd640 -1 bdev(0x55e0bff5d900 /var/lib/ceph/osd/ceph-4/block) open stat got: (1) Operation not permitted 2024-08-14T06:51:57.529 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:51:57.527+0000 7f7f9f6dd640 -1 bluestore(/var/lib/ceph/osd/ceph-4) _read_fsid unparsable uuid 2024-08-14T06:52:00.701 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-08-14T06:52:00.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 5 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:52:00.834 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:00.832+0000 7f53474ff640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-5/keyring: can't open /var/lib/ceph/osd/ceph-5/keyring: (2) No such file or directory 2024-08-14T06:52:00.834 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:00.833+0000 7f53474ff640 -1 created new key in keyring /var/lib/ceph/osd/ceph-5/keyring 2024-08-14T06:52:00.834 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:00.833+0000 7f53474ff640 -1 bdev(0x5565c82fb900 /var/lib/ceph/osd/ceph-5/block) open stat got: (1) Operation not permitted 2024-08-14T06:52:00.834 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:00.833+0000 7f53474ff640 -1 bluestore(/var/lib/ceph/osd/ceph-5) _read_fsid unparsable uuid 2024-08-14T06:52:03.682 DEBUG:teuthology.orchestra.run.smithi098:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-08-14T06:52:03.686 DEBUG:teuthology.orchestra.run.smithi191:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-08-14T06:52:03.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-14T06:52:03.785 DEBUG:teuthology.orchestra.run.smithi191:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 6 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:52:03.884 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:03.882+0000 7f4a32018640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-6/keyring: can't open /var/lib/ceph/osd/ceph-6/keyring: (2) No such file or directory 2024-08-14T06:52:03.884 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:03.883+0000 7f4a32018640 -1 created new key in keyring /var/lib/ceph/osd/ceph-6/keyring 2024-08-14T06:52:03.884 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:03.883+0000 7f4a32018640 -1 bdev(0x55a774c49900 /var/lib/ceph/osd/ceph-6/block) open stat got: (1) Operation not permitted 2024-08-14T06:52:03.884 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:03.883+0000 7f4a32018640 -1 bluestore(/var/lib/ceph/osd/ceph-6) _read_fsid unparsable uuid 2024-08-14T06:52:07.288 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-14T06:52:07.316 DEBUG:teuthology.orchestra.run.smithi191:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 7 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:52:07.415 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:07.414+0000 7fed7b62e640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-7/keyring: can't open /var/lib/ceph/osd/ceph-7/keyring: (2) No such file or directory 2024-08-14T06:52:07.415 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:07.414+0000 7fed7b62e640 -1 created new key in keyring /var/lib/ceph/osd/ceph-7/keyring 2024-08-14T06:52:07.415 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:07.414+0000 7fed7b62e640 -1 bdev(0x56406b69d900 /var/lib/ceph/osd/ceph-7/block) open stat got: (1) Operation not permitted 2024-08-14T06:52:07.415 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:07.414+0000 7fed7b62e640 -1 bluestore(/var/lib/ceph/osd/ceph-7) _read_fsid unparsable uuid 2024-08-14T06:52:10.828 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-14T06:52:10.857 INFO:tasks.ceph:Reading keys from all nodes... 2024-08-14T06:52:10.857 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:10.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/mgr/ceph-y/keyring of=/dev/stdout 2024-08-14T06:52:10.887 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:10.887 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/mds/ceph-a/keyring of=/dev/stdout 2024-08-14T06:52:10.956 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:10.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/mds/ceph-c/keyring of=/dev/stdout 2024-08-14T06:52:11.023 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:11.024 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-0/keyring of=/dev/stdout 2024-08-14T06:52:11.091 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:11.091 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-1/keyring of=/dev/stdout 2024-08-14T06:52:11.158 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:11.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-2/keyring of=/dev/stdout 2024-08-14T06:52:11.225 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:11.225 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-3/keyring of=/dev/stdout 2024-08-14T06:52:11.291 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:11.291 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/mgr/ceph-x/keyring of=/dev/stdout 2024-08-14T06:52:11.321 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:11.322 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/mds/ceph-b/keyring of=/dev/stdout 2024-08-14T06:52:11.391 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:11.391 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/mds/ceph-d/keyring of=/dev/stdout 2024-08-14T06:52:11.460 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:11.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-4/keyring of=/dev/stdout 2024-08-14T06:52:11.527 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:11.527 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-5/keyring of=/dev/stdout 2024-08-14T06:52:11.594 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:11.595 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-6/keyring of=/dev/stdout 2024-08-14T06:52:11.661 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:11.661 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-7/keyring of=/dev/stdout 2024-08-14T06:52:11.727 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:11.728 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/etc/ceph/ceph.client.0.keyring of=/dev/stdout 2024-08-14T06:52:11.747 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:11.747 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/etc/ceph/ceph.client.2.keyring of=/dev/stdout 2024-08-14T06:52:11.805 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:11.805 DEBUG:teuthology.orchestra.run.smithi191:> dd if=/etc/ceph/ceph.client.1.keyring of=/dev/stdout 2024-08-14T06:52:11.821 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:11.821 DEBUG:teuthology.orchestra.run.smithi191:> dd if=/etc/ceph/ceph.client.3.keyring of=/dev/stdout 2024-08-14T06:52:11.878 INFO:tasks.ceph:Adding keys to all mons... 2024-08-14T06:52:11.878 DEBUG:teuthology.orchestra.run.smithi098:> sudo tee -a /etc/ceph/ceph.keyring 2024-08-14T06:52:11.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo tee -a /etc/ceph/ceph.keyring 2024-08-14T06:52:11.921 INFO:teuthology.orchestra.run.smithi098.stdout:[mgr.y] 2024-08-14T06:52:11.921 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD4U7xmB4WTChAA6cRNHQeE/0yqIBMs1oHtMA== 2024-08-14T06:52:11.921 INFO:teuthology.orchestra.run.smithi098.stdout:[mds.a] 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD4U7xm7IDfEhAADnWWY4C1yTD5Lr5FTjj6Lw== 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout:[mds.c] 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD4U7xmr3N0GhAARr27YV2Lg2RE1ZkKW+uu5g== 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.0] 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD8U7xmOWQ0DBAAhoqSMrTl8pnX2g684cXjRQ== 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.1] 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD/U7xm1/IPLRAAPXveeY99wY7Js/TyXPeV7w== 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.2] 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQADVLxmowE1EhAAtaQleBNMYsj5YR9C1CzbVQ== 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.3] 2024-08-14T06:52:11.922 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQAGVLxmsIipMhAAlXkU0wf8/9/M6vAOD9zLGg== 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout:[mgr.x] 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD4U7xm69IKDxAA/WcJVmDjN82xIDptEKUXBA== 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout:[mds.b] 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD4U7xm0yPAHxAAtYhUkirju8cwxr2FAF9lNQ== 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout:[mds.d] 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD4U7xmKwhnJxAAl7cWfjbhaXm4FxZQWdCu8Q== 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.4] 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQANVLxmpCqCHxAAzmNU92bZ129mlKkyt6N33w== 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.5] 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQAQVLxmlfa0MRAADY3k2NLXy8nEz24uk4ss7Q== 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.6] 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQATVLxmM2CvNBAAhH7M9+1Py/8UseEm5C1n1Q== 2024-08-14T06:52:11.923 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.7] 2024-08-14T06:52:11.924 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQAXVLxmVxDCGBAAPtXnCddMSbNUDii7ym9uXw== 2024-08-14T06:52:11.924 INFO:teuthology.orchestra.run.smithi098.stdout:[client.0] 2024-08-14T06:52:11.924 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD4U7xmHIYkLBAAWu6ZBkVj86Etj8Rd/QqoZA== 2024-08-14T06:52:11.924 INFO:teuthology.orchestra.run.smithi098.stdout:[client.2] 2024-08-14T06:52:11.924 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD4U7xmonLxLxAAgpqQ5dJC6i6r8Wm7qD18Tg== 2024-08-14T06:52:11.924 INFO:teuthology.orchestra.run.smithi098.stdout:[client.1] 2024-08-14T06:52:11.924 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD4U7xmqOLMMxAAajvIrjyM1Mw+HRIazVpC5A== 2024-08-14T06:52:11.924 INFO:teuthology.orchestra.run.smithi098.stdout:[client.3] 2024-08-14T06:52:11.924 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQD4U7xm7+ahNxAA1NE7vhcjYyqddqh1YwFPrA== 2024-08-14T06:52:11.941 INFO:teuthology.orchestra.run.smithi191.stdout:[mgr.y] 2024-08-14T06:52:11.941 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD4U7xmB4WTChAA6cRNHQeE/0yqIBMs1oHtMA== 2024-08-14T06:52:11.942 INFO:teuthology.orchestra.run.smithi191.stdout:[mds.a] 2024-08-14T06:52:11.942 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD4U7xm7IDfEhAADnWWY4C1yTD5Lr5FTjj6Lw== 2024-08-14T06:52:11.942 INFO:teuthology.orchestra.run.smithi191.stdout:[mds.c] 2024-08-14T06:52:11.942 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD4U7xmr3N0GhAARr27YV2Lg2RE1ZkKW+uu5g== 2024-08-14T06:52:11.942 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.0] 2024-08-14T06:52:11.942 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD8U7xmOWQ0DBAAhoqSMrTl8pnX2g684cXjRQ== 2024-08-14T06:52:11.942 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.1] 2024-08-14T06:52:11.942 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD/U7xm1/IPLRAAPXveeY99wY7Js/TyXPeV7w== 2024-08-14T06:52:11.942 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.2] 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQADVLxmowE1EhAAtaQleBNMYsj5YR9C1CzbVQ== 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.3] 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQAGVLxmsIipMhAAlXkU0wf8/9/M6vAOD9zLGg== 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout:[mgr.x] 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD4U7xm69IKDxAA/WcJVmDjN82xIDptEKUXBA== 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout:[mds.b] 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD4U7xm0yPAHxAAtYhUkirju8cwxr2FAF9lNQ== 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout:[mds.d] 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD4U7xmKwhnJxAAl7cWfjbhaXm4FxZQWdCu8Q== 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.4] 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQANVLxmpCqCHxAAzmNU92bZ129mlKkyt6N33w== 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.5] 2024-08-14T06:52:11.943 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQAQVLxmlfa0MRAADY3k2NLXy8nEz24uk4ss7Q== 2024-08-14T06:52:11.944 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.6] 2024-08-14T06:52:11.944 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQATVLxmM2CvNBAAhH7M9+1Py/8UseEm5C1n1Q== 2024-08-14T06:52:11.944 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.7] 2024-08-14T06:52:11.944 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQAXVLxmVxDCGBAAPtXnCddMSbNUDii7ym9uXw== 2024-08-14T06:52:11.944 INFO:teuthology.orchestra.run.smithi191.stdout:[client.0] 2024-08-14T06:52:11.944 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD4U7xmHIYkLBAAWu6ZBkVj86Etj8Rd/QqoZA== 2024-08-14T06:52:11.944 INFO:teuthology.orchestra.run.smithi191.stdout:[client.2] 2024-08-14T06:52:11.944 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD4U7xmonLxLxAAgpqQ5dJC6i6r8Wm7qD18Tg== 2024-08-14T06:52:11.945 INFO:teuthology.orchestra.run.smithi191.stdout:[client.1] 2024-08-14T06:52:11.945 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD4U7xmqOLMMxAAajvIrjyM1Mw+HRIazVpC5A== 2024-08-14T06:52:11.945 INFO:teuthology.orchestra.run.smithi191.stdout:[client.3] 2024-08-14T06:52:11.945 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQD4U7xm7+ahNxAA1NE7vhcjYyqddqh1YwFPrA== 2024-08-14T06:52:11.946 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-08-14T06:52:11.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-08-14T06:52:12.048 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.a --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-08-14T06:52:12.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.a --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-08-14T06:52:12.115 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.c --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-08-14T06:52:12.118 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.c --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-08-14T06:52:12.181 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.244 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.308 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.311 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.372 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.436 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-08-14T06:52:12.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-08-14T06:52:12.500 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.b --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-08-14T06:52:12.502 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.b --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-08-14T06:52:12.567 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.d --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-08-14T06:52:12.569 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.d --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-08-14T06:52:12.633 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.696 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.760 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.824 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-14T06:52:12.887 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-14T06:52:12.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-14T06:52:12.954 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-14T06:52:12.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-14T06:52:13.021 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-14T06:52:13.025 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-14T06:52:13.089 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-14T06:52:13.091 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-14T06:52:13.155 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-08-14T06:52:13.156 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/mon/ceph-a 2024-08-14T06:52:13.180 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i a --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-08-14T06:52:13.287 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-a 2024-08-14T06:52:13.312 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/mon/ceph-b 2024-08-14T06:52:13.337 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i b --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-08-14T06:52:13.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-b 2024-08-14T06:52:13.470 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/mon/ceph-c 2024-08-14T06:52:13.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i c --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-08-14T06:52:13.643 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-c 2024-08-14T06:52:13.667 DEBUG:teuthology.orchestra.run.smithi098:> rm -- /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:52:13.669 DEBUG:teuthology.orchestra.run.smithi191:> rm -- /home/ubuntu/cephtest/ceph.monmap 2024-08-14T06:52:13.723 INFO:tasks.ceph:Starting mon daemons in cluster ceph... 2024-08-14T06:52:13.723 INFO:tasks.ceph.mon.a:Restarting daemon 2024-08-14T06:52:13.723 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-08-14T06:52:13.726 INFO:tasks.ceph.mon.a:Started 2024-08-14T06:52:13.726 INFO:tasks.ceph.mon.b:Restarting daemon 2024-08-14T06:52:13.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-08-14T06:52:13.765 INFO:tasks.ceph.mon.b:Started 2024-08-14T06:52:13.765 INFO:tasks.ceph.mon.c:Restarting daemon 2024-08-14T06:52:13.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-08-14T06:52:13.767 INFO:tasks.ceph.mon.c:Started 2024-08-14T06:52:13.767 INFO:tasks.ceph:Starting mgr daemons in cluster ceph... 2024-08-14T06:52:13.767 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-08-14T06:52:13.767 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-08-14T06:52:13.769 INFO:tasks.ceph.mgr.y:Started 2024-08-14T06:52:13.770 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-08-14T06:52:13.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-08-14T06:52:13.772 INFO:tasks.ceph.mgr.x:Started 2024-08-14T06:52:13.772 INFO:tasks.ceph:setting config [client] client_mount_timeout = 600 2024-08-14T06:52:13.772 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set client client_mount_timeout 600 2024-08-14T06:52:13.774 INFO:tasks.ceph:setting config [client] debug_client = 20 2024-08-14T06:52:13.774 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set client debug_client 20 2024-08-14T06:52:13.817 INFO:tasks.ceph:setting config [client] debug_ms = 1 2024-08-14T06:52:13.818 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set client debug_ms 1 2024-08-14T06:52:13.860 INFO:tasks.ceph:setting config [client] rados_mon_op_timeout = 900 2024-08-14T06:52:13.860 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set client rados_mon_op_timeout 900 2024-08-14T06:52:13.903 INFO:tasks.ceph:setting config [client] rados_osd_op_timeout = 900 2024-08-14T06:52:13.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set client rados_osd_op_timeout 900 2024-08-14T06:52:13.946 INFO:tasks.ceph:setting config [mds] debug_mds = 20 2024-08-14T06:52:13.947 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mds debug_mds 20 2024-08-14T06:52:13.989 INFO:tasks.ceph:setting config [mds] debug_mds_balancer = 20 2024-08-14T06:52:13.989 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mds debug_mds_balancer 20 2024-08-14T06:52:14.033 INFO:tasks.ceph:setting config [mds] debug_ms = 2 2024-08-14T06:52:14.033 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mds debug_ms 2 2024-08-14T06:52:14.077 INFO:tasks.ceph:setting config [mds] mds_debug_frag = True 2024-08-14T06:52:14.078 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mds mds_debug_frag True 2024-08-14T06:52:14.124 INFO:tasks.ceph:setting config [mds] mds_debug_scatterstat = True 2024-08-14T06:52:14.125 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mds mds_debug_scatterstat True 2024-08-14T06:52:14.167 INFO:tasks.ceph:setting config [mds] mds_op_complaint_time = 180 2024-08-14T06:52:14.168 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mds mds_op_complaint_time 180 2024-08-14T06:52:14.212 INFO:tasks.ceph:setting config [mds] mds_verify_scatter = True 2024-08-14T06:52:14.212 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mds mds_verify_scatter True 2024-08-14T06:52:14.237 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.236+0000 7f91237fe640 1 -- 172.21.15.98:0/1601202805 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f91381513b0 msgr2=0x7f913815a800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.237 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.236+0000 7f91237fe640 1 --2- 172.21.15.98:0/1601202805 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f91381513b0 0x7f913815a800 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f9128000f60 tx=0x7f912800ee10 comp rx=0 tx=0).stop 2024-08-14T06:52:14.240 INFO:tasks.ceph:setting config [mds] osd_op_complaint_time = 180 2024-08-14T06:52:14.240 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mds osd_op_complaint_time 180 2024-08-14T06:52:14.241 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.237+0000 7f91237fe640 1 -- 172.21.15.98:0/1601202805 shutdown_connections 2024-08-14T06:52:14.241 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.237+0000 7f91237fe640 1 --2- 172.21.15.98:0/1601202805 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f913815ad40 0x7f9138142360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.241 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.238+0000 7f91237fe640 1 --2- 172.21.15.98:0/1601202805 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f91381513b0 0x7f913815a800 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.241 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.238+0000 7f91237fe640 1 --2- 172.21.15.98:0/1601202805 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f913806f7b0 0x7f913813f9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.241 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.238+0000 7f91237fe640 1 -- 172.21.15.98:0/1601202805 >> 172.21.15.98:0/1601202805 conn(0x7f913806d650 msgr2=0x7f913806f0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.241 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.239+0000 7f91237fe640 1 -- 172.21.15.98:0/1601202805 shutdown_connections 2024-08-14T06:52:14.241 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.239+0000 7f91237fe640 1 -- 172.21.15.98:0/1601202805 wait complete. 2024-08-14T06:52:14.250 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.250+0000 7fda597fa640 1 -- 172.21.15.98:0/844859579 <== mon.1 v2:172.21.15.191:3300/0 9 ==== config(3 keys) ==== 79+0+0 (secure 0 0 0) 0x7fda4801ac10 con 0x7fda5c14fd40 2024-08-14T06:52:14.252 INFO:tasks.ceph:setting config [mds] rados_mon_op_timeout = 900 2024-08-14T06:52:14.252 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mds rados_mon_op_timeout 900 2024-08-14T06:52:14.254 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.254+0000 7fdc85cf9640 1 -- 172.21.15.98:0/3091478896 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdc60005080 con 0x7fdc8015efb0 2024-08-14T06:52:14.256 INFO:tasks.ceph:setting config [mds] rados_osd_op_timeout = 900 2024-08-14T06:52:14.256 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mds rados_osd_op_timeout 900 2024-08-14T06:52:14.256 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.254+0000 7ff0ab7fe640 1 -- 172.21.15.98:0/3963056040 <== mon.1 v2:172.21.15.191:3300/0 9 ==== config(3 keys) ==== 79+0+0 (secure 0 0 0) 0x7ff0b80051c0 con 0x7ff0c41627f0 2024-08-14T06:52:14.258 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.257+0000 7fdc67fff640 1 -- 172.21.15.98:0/3091478896 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7fdc7804c020 con 0x7fdc8015efb0 2024-08-14T06:52:14.259 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.257+0000 7fbd65ffb640 1 -- 172.21.15.98:0/3233458341 <== mon.0 v2:172.21.15.98:3300/0 9 ==== config(3 keys) ==== 79+0+0 (secure 0 0 0) 0x7fbd600039d0 con 0x7fbd700d2a00 2024-08-14T06:52:14.262 INFO:tasks.ceph:setting config [mon] mon_op_complaint_time = 120 2024-08-14T06:52:14.262 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mon mon_op_complaint_time 120 2024-08-14T06:52:14.263 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.260+0000 7fda597fa640 1 -- 172.21.15.98:0/844859579 <== mon.1 v2:172.21.15.191:3300/0 10 ==== mon_command_ack([{prefix=config set, name=debug_client}]=0 v4)=0 v4) ==== 114+0+0 (secure 0 0 0) 0x7fda480035f0 con 0x7fda5c14fd40 2024-08-14T06:52:14.265 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.263+0000 7fda62c84640 1 -- 172.21.15.98:0/844859579 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fda5c14fd40 msgr2=0x7fda5c150be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.265 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.264+0000 7fda62c84640 1 --2- 172.21.15.98:0/844859579 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fda5c14fd40 0x7fda5c150be0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fda4800f780 tx=0x7fda4800fc40 comp rx=0 tx=0).stop 2024-08-14T06:52:14.265 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.264+0000 7fda62c84640 1 -- 172.21.15.98:0/844859579 shutdown_connections 2024-08-14T06:52:14.265 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.264+0000 7fda62c84640 1 --2- 172.21.15.98:0/844859579 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fda5c1f3120 0x7fda5c14bd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.265 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.264+0000 7fda62c84640 1 --2- 172.21.15.98:0/844859579 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fda5c14fd40 0x7fda5c150be0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.266 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.264+0000 7fda62c84640 1 --2- 172.21.15.98:0/844859579 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fda4c005150 0x7fda5c153080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.266 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.264+0000 7fda62c84640 1 -- 172.21.15.98:0/844859579 >> 172.21.15.98:0/844859579 conn(0x7fda5c06d650 msgr2=0x7fda5c0b4690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.270 INFO:tasks.ceph:setting config [mon] mon_warn_on_pool_no_app = False 2024-08-14T06:52:14.270 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set mon mon_warn_on_pool_no_app False 2024-08-14T06:52:14.271 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.264+0000 7fda62c84640 1 -- 172.21.15.98:0/844859579 shutdown_connections 2024-08-14T06:52:14.271 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.264+0000 7fda62c84640 1 -- 172.21.15.98:0/844859579 wait complete. 2024-08-14T06:52:14.271 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.265+0000 7fbd765f1640 1 -- 172.21.15.98:0/3233458341 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command([{prefix=config set, name=rados_osd_op_timeout}] v 0) -- 0x7fbd4c005100 con 0x7fbd700d2a00 2024-08-14T06:52:14.272 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.266+0000 7f2c85326640 1 -- 172.21.15.98:0/3265477866 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2c80143810 con 0x7f2c801f45a0 2024-08-14T06:52:14.276 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.275+0000 7f2c677fe640 1 -- 172.21.15.98:0/3265477866 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f2c7001f020 con 0x7f2c801f45a0 2024-08-14T06:52:14.278 INFO:tasks.ceph:setting config [osd] osd_op_complaint_time = 180 2024-08-14T06:52:14.278 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph config set osd osd_op_complaint_time 180 2024-08-14T06:52:14.287 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.287+0000 7fbd65ffb640 1 -- 172.21.15.98:0/3233458341 <== mon.0 v2:172.21.15.98:3300/0 10 ==== mon_command_ack([{prefix=config set, name=rados_osd_op_timeout}]=0 v5)=0 v5) ==== 123+0+0 (secure 0 0 0) 0x7fbd60003ce0 con 0x7fbd700d2a00 2024-08-14T06:52:14.290 DEBUG:tasks.ceph:set 18 configs 2024-08-14T06:52:14.291 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'client_mount_timeout', '600'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.291 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'debug_client', '20'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.292 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.287+0000 7fdc67fff640 1 -- 172.21.15.98:0/3091478896 <== mon.2 v2:172.21.15.191:3301/0 8 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7fdc780098e0 con 0x7fdc8015efb0 2024-08-14T06:52:14.292 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.289+0000 7ff0ab7fe640 1 -- 172.21.15.98:0/3963056040 <== mon.1 v2:172.21.15.191:3300/0 10 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7ff0b80053c0 con 0x7ff0c41627f0 2024-08-14T06:52:14.292 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.288+0000 7f2c677fe640 1 -- 172.21.15.98:0/3265477866 <== mon.0 v2:172.21.15.98:3300/0 7 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f2c7000bdc0 con 0x7f2c801f45a0 2024-08-14T06:52:14.293 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.290+0000 7fbd65ffb640 1 -- 172.21.15.98:0/3233458341 <== mon.0 v2:172.21.15.98:3300/0 11 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7fbd6001a5a0 con 0x7fbd700d2a00 2024-08-14T06:52:14.303 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.302+0000 7fbd765f1640 1 -- 172.21.15.98:0/3233458341 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fbd700d2a00 msgr2=0x7fbd701f8400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.303 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.302+0000 7fbd765f1640 1 --2- 172.21.15.98:0/3233458341 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fbd700d2a00 0x7fbd701f8400 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7fbd6000c540 tx=0x7fbd6000ca00 comp rx=0 tx=0).stop 2024-08-14T06:52:14.303 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.302+0000 7fbd765f1640 1 -- 172.21.15.98:0/3233458341 shutdown_connections 2024-08-14T06:52:14.303 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.302+0000 7fbd765f1640 1 --2- 172.21.15.98:0/3233458341 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fbd7009ee40 0x7fbd701e4070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.303 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.302+0000 7fbd765f1640 1 --2- 172.21.15.98:0/3233458341 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fbd700d2a00 0x7fbd701f8400 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.303 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.302+0000 7fbd765f1640 1 --2- 172.21.15.98:0/3233458341 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fbd700d0c10 0x7fbd7009d4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.303 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.302+0000 7fbd765f1640 1 -- 172.21.15.98:0/3233458341 >> 172.21.15.98:0/3233458341 conn(0x7fbd700d1ee0 msgr2=0x7fbd700af4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.304 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'debug_ms', '1'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.304 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'rados_mon_op_timeout', '900'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.308 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.308+0000 7fbd765f1640 1 -- 172.21.15.98:0/3233458341 shutdown_connections 2024-08-14T06:52:14.308 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.308+0000 7fbd765f1640 1 -- 172.21.15.98:0/3233458341 wait complete. 2024-08-14T06:52:14.327 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.326+0000 7f4f36c09640 1 -- 172.21.15.98:0/984452538 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f4f300cdf40 msgr2=0x7f4f300ce350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.327 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.326+0000 7f4f36c09640 1 --2- 172.21.15.98:0/984452538 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f4f300cdf40 0x7f4f300ce350 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f4f300cdc30 tx=0x7f4f2802b580 comp rx=0 tx=0).stop 2024-08-14T06:52:14.331 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.327+0000 7f4f36c09640 1 -- 172.21.15.98:0/984452538 shutdown_connections 2024-08-14T06:52:14.331 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.327+0000 7f4f36c09640 1 --2- 172.21.15.98:0/984452538 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f4f300cdf40 0x7f4f300ce350 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.331 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.327+0000 7f4f36c09640 1 --2- 172.21.15.98:0/984452538 >> v2:172.21.15.191:3300/0 conn(0x7f4f30070b80 0x7f4f3016f7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.331 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.328+0000 7f4f36c09640 1 -- 172.21.15.98:0/984452538 >> 172.21.15.98:0/984452538 conn(0x7f4f300cfee0 msgr2=0x7f4f300d02f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.332 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.328+0000 7f4f36c09640 1 -- 172.21.15.98:0/984452538 shutdown_connections 2024-08-14T06:52:14.332 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.328+0000 7f4f36c09640 1 -- 172.21.15.98:0/984452538 wait complete. 2024-08-14T06:52:14.333 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.333+0000 7f4f36c09640 1 Processor -- start 2024-08-14T06:52:14.336 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.333+0000 7f4f36c09640 1 -- start start 2024-08-14T06:52:14.337 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.337+0000 7f4f36c09640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f4f30070b80 0x7f4f300b2e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.343 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.342+0000 7f4f349a5640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f4f30070b80 0x7f4f300b2e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.343 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.342+0000 7f4f349a5640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f4f30070b80 0x7f4f300b2e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:40154/0 (socket says 172.21.15.98:40154) 2024-08-14T06:52:14.343 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.342+0000 7f4f349a5640 1 -- 172.21.15.98:0/3923728278 learned_addr learned my addr 172.21.15.98:0/3923728278 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.343 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.342+0000 7f4f36c09640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f4f3008b630 0x7f4f300c9040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.344 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.342+0000 7f4f36c09640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f4f300cdf40 0x7f4f300c9580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.344 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.343+0000 7f4f36c09640 1 -- 172.21.15.98:0/3923728278 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f4f300cea10 con 0x7f4f30070b80 2024-08-14T06:52:14.344 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.343+0000 7f4f351a6640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f4f300cdf40 0x7f4f300c9580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.344 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.343+0000 7f4f36c09640 1 -- 172.21.15.98:0/3923728278 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f4f300ce890 con 0x7f4f3008b630 2024-08-14T06:52:14.344 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.343+0000 7f4f36c09640 1 -- 172.21.15.98:0/3923728278 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f4f300ceb90 con 0x7f4f300cdf40 2024-08-14T06:52:14.345 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.344+0000 7f4f351a6640 1 -- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f4f3008b630 msgr2=0x7f4f300c9040 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-08-14T06:52:14.345 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.344+0000 7f4f2ffff640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f4f3008b630 0x7f4f300c9040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.345 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.344+0000 7f4f351a6640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f4f3008b630 0x7f4f300c9040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.345 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.344+0000 7f4f351a6640 1 -- 172.21.15.98:0/3923728278 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f4f30070b80 msgr2=0x7f4f300b2e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.345 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.344+0000 7f4f351a6640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f4f30070b80 0x7f4f300b2e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.345 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.344+0000 7f4f351a6640 1 -- 172.21.15.98:0/3923728278 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4f300c7fa0 con 0x7f4f300cdf40 2024-08-14T06:52:14.345 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.344+0000 7f4f349a5640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f4f30070b80 0x7f4f300b2e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:14.350 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.350+0000 7f4f351a6640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f4f300cdf40 0x7f4f300c9580 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f4f300ca4e0 tx=0x7f4f28033640 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.350+0000 7f4f2d7fa640 1 -- 172.21.15.98:0/3923728278 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f28041030 con 0x7f4f300cdf40 2024-08-14T06:52:14.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.351+0000 7f4f2d7fa640 1 -- 172.21.15.98:0/3923728278 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f4f28003970 con 0x7f4f300cdf40 2024-08-14T06:52:14.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.351+0000 7f4f36c09640 1 -- 172.21.15.98:0/3923728278 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4f300c8280 con 0x7f4f300cdf40 2024-08-14T06:52:14.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.351+0000 7f4f36c09640 1 -- 172.21.15.98:0/3923728278 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f4f300c8840 con 0x7f4f300cdf40 2024-08-14T06:52:14.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.351+0000 7f4f2d7fa640 1 -- 172.21.15.98:0/3923728278 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f28037040 con 0x7f4f300cdf40 2024-08-14T06:52:14.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.351+0000 7f4f2d7fa640 1 -- 172.21.15.98:0/3923728278 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f4f280337b0 con 0x7f4f300cdf40 2024-08-14T06:52:14.356 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.351+0000 7f4f2d7fa640 1 -- 172.21.15.98:0/3923728278 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f4f28004070 con 0x7f4f300cdf40 2024-08-14T06:52:14.357 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.357+0000 7f4f36c09640 1 -- 172.21.15.98:0/3923728278 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4f10005080 con 0x7f4f300cdf40 2024-08-14T06:52:14.364 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.359+0000 7f4f2d7fa640 1 -- 172.21.15.98:0/3923728278 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f4f28007830 con 0x7f4f300cdf40 2024-08-14T06:52:14.364 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.360+0000 7f008208d640 1 -- 172.21.15.98:0/1138419540 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f007c0aea60 msgr2=0x7f007c0aee70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.365 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.360+0000 7f008208d640 1 --2- 172.21.15.98:0/1138419540 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f007c0aea60 0x7f007c0aee70 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f007c0b27c0 tx=0x7f007402a820 comp rx=0 tx=0).stop 2024-08-14T06:52:14.365 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.361+0000 7f008188c640 1 --2- 172.21.15.98:0/1138419540 >> v2:172.21.15.191:3300/0 conn(0x7f007c06ed80 0x7f007c06f190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-08-14T06:52:14.367 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.361+0000 7f008208d640 1 -- 172.21.15.98:0/1138419540 shutdown_connections 2024-08-14T06:52:14.367 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.361+0000 7f008208d640 1 --2- 172.21.15.98:0/1138419540 >> v2:172.21.15.191:3300/0 conn(0x7f007c06ed80 0x7f007c06f190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.367 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.361+0000 7f008208d640 1 --2- 172.21.15.98:0/1138419540 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f007c0aea60 0x7f007c0aee70 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.367 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.361+0000 7f008208d640 1 --2- 172.21.15.98:0/1138419540 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f007c0cda80 0x7f007c0ae520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.367 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.361+0000 7f008208d640 1 -- 172.21.15.98:0/1138419540 >> 172.21.15.98:0/1138419540 conn(0x7f007c0b1e60 msgr2=0x7f007c0cfca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.367 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.361+0000 7f008208d640 1 -- 172.21.15.98:0/1138419540 shutdown_connections 2024-08-14T06:52:14.367 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.361+0000 7f008208d640 1 -- 172.21.15.98:0/1138419540 wait complete. 2024-08-14T06:52:14.367 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.364+0000 7f008208d640 1 Processor -- start 2024-08-14T06:52:14.375 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.366+0000 7f008208d640 1 -- start start 2024-08-14T06:52:14.375 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.368+0000 7f008208d640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f007c06ed80 0x7f007c157170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.368+0000 7f008208d640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f007c0aea60 0x7f007c158fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.368+0000 7f008208d640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f007c0cda80 0x7f007c15b8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.368+0000 7f008208d640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f007c06fa60 con 0x7f007c0aea60 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.368+0000 7f008208d640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f007c06f8e0 con 0x7f007c06ed80 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.368+0000 7f008208d640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f007c06fbe0 con 0x7f007c0cda80 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.371+0000 7f008088a640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f007c0aea60 0x7f007c158fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.372+0000 7f008088a640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f007c0aea60 0x7f007c158fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:40174/0 (socket says 172.21.15.98:40174) 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.372+0000 7f008088a640 1 -- 172.21.15.98:0/3744203810 learned_addr learned my addr 172.21.15.98:0/3744203810 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.372+0000 7f008108b640 1 --2- 172.21.15.98:0/3744203810 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f007c06ed80 0x7f007c157170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.372+0000 7f008088a640 1 -- 172.21.15.98:0/3744203810 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f007c0cda80 msgr2=0x7f007c15b8d0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.372+0000 7f008088a640 1 --2- 172.21.15.98:0/3744203810 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f007c0cda80 0x7f007c15b8d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.372+0000 7f008088a640 1 -- 172.21.15.98:0/3744203810 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f007c06ed80 msgr2=0x7f007c157170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.376 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.372+0000 7f008088a640 1 --2- 172.21.15.98:0/3744203810 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f007c06ed80 0x7f007c157170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.377 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.372+0000 7f008088a640 1 -- 172.21.15.98:0/3744203810 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f007c1594e0 con 0x7f007c0aea60 2024-08-14T06:52:14.377 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.373+0000 7f008088a640 1 --2- 172.21.15.98:0/3744203810 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f007c0aea60 0x7f007c158fa0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f0074009cf0 tx=0x7f0074032f70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.377 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.374+0000 7f0071ffb640 1 -- 172.21.15.98:0/3744203810 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0074041020 con 0x7f007c0aea60 2024-08-14T06:52:14.377 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.374+0000 7f0071ffb640 1 -- 172.21.15.98:0/3744203810 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f0074038040 con 0x7f007c0aea60 2024-08-14T06:52:14.377 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.374+0000 7f0071ffb640 1 -- 172.21.15.98:0/3744203810 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0074037030 con 0x7f007c0aea60 2024-08-14T06:52:14.378 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.375+0000 7f008208d640 1 -- 172.21.15.98:0/3744203810 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f007c0cbea0 con 0x7f007c0aea60 2024-08-14T06:52:14.378 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.375+0000 7f008208d640 1 -- 172.21.15.98:0/3744203810 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f007c0cc460 con 0x7f007c0aea60 2024-08-14T06:52:14.379 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.378+0000 7f0071ffb640 1 -- 172.21.15.98:0/3744203810 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f0074044020 con 0x7f007c0aea60 2024-08-14T06:52:14.379 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.378+0000 7f0071ffb640 1 -- 172.21.15.98:0/3744203810 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f00740052a0 con 0x7f007c0aea60 2024-08-14T06:52:14.379 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.379+0000 7f00677fe640 1 -- 172.21.15.98:0/3744203810 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f007c0aee70 con 0x7f007c0aea60 2024-08-14T06:52:14.385 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.384+0000 7f0071ffb640 1 -- 172.21.15.98:0/3744203810 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f007404b070 con 0x7f007c0aea60 2024-08-14T06:52:14.462 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.461+0000 7f7c9f59e640 1 -- 172.21.15.98:0/3547711289 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7ca00cda80 msgr2=0x7f7c88004820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.462 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.461+0000 7f7c9f59e640 1 --2- 172.21.15.98:0/3547711289 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7ca00cda80 0x7f7c88004820 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f7c9800c610 tx=0x7f7c88005830 comp rx=0 tx=0).stop 2024-08-14T06:52:14.462 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.461+0000 7f7c9ed9d640 1 -- 172.21.15.98:0/3547711289 reap_dead start 2024-08-14T06:52:14.463 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.462+0000 7f7c9f59e640 1 -- 172.21.15.98:0/3547711289 shutdown_connections 2024-08-14T06:52:14.463 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.462+0000 7f7c9f59e640 1 -- 172.21.15.98:0/3547711289 >> 172.21.15.98:0/3547711289 conn(0x7f7ca00b1e60 msgr2=0x7f7ca00cfca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.473+0000 7f7c9f59e640 1 -- 172.21.15.98:0/3547711289 shutdown_connections 2024-08-14T06:52:14.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.479+0000 7f7c9f59e640 1 -- 172.21.15.98:0/3547711289 wait complete. 2024-08-14T06:52:14.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.482+0000 7f7c9f59e640 1 Processor -- start 2024-08-14T06:52:14.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.482+0000 7f7c9f59e640 1 -- start start 2024-08-14T06:52:14.485 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.485+0000 7f7c9f59e640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7ca00b4860 0x7f7ca00cd0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.485 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.485+0000 7f7c9f59e640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7ca02017b0 0x7f7ca01525a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.485 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.485+0000 7f7c9f59e640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7ca0152ae0 0x7f7ca0149730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.486 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.485+0000 7f7c9ed9d640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7ca0152ae0 0x7f7ca0149730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.486 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.485+0000 7f7c9ed9d640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7ca0152ae0 0x7f7ca0149730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:40202/0 (socket says 172.21.15.98:40202) 2024-08-14T06:52:14.486 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.486+0000 7f7c9ed9d640 1 -- 172.21.15.98:0/623982275 learned_addr learned my addr 172.21.15.98:0/623982275 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.487 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.486+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f7ca00ae6a0 con 0x7f7ca0152ae0 2024-08-14T06:52:14.487 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.486+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f7ca00ae520 con 0x7f7ca00b4860 2024-08-14T06:52:14.487 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.486+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f7ca00ae820 con 0x7f7ca02017b0 2024-08-14T06:52:14.487 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.486+0000 7f7c9ed9d640 1 -- 172.21.15.98:0/623982275 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7ca02017b0 msgr2=0x7f7ca01525a0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-08-14T06:52:14.488 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.486+0000 7f7c9ed9d640 1 --2- 172.21.15.98:0/623982275 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7ca02017b0 0x7f7ca01525a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.488 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.486+0000 7f7c9ed9d640 1 -- 172.21.15.98:0/623982275 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7ca00b4860 msgr2=0x7f7ca00cd0a0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-08-14T06:52:14.488 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.486+0000 7f7c9ed9d640 1 --2- 172.21.15.98:0/623982275 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7ca00b4860 0x7f7ca00cd0a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.488 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.486+0000 7f7c9ed9d640 1 -- 172.21.15.98:0/623982275 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7ca0152ef0 con 0x7f7ca0152ae0 2024-08-14T06:52:14.489 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.487+0000 7f7c9ed9d640 1 --2- 172.21.15.98:0/623982275 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7ca0152ae0 0x7f7ca0149730 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f7ca00b27f0 tx=0x7f7c900016b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.489 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.488+0000 7f7c8effd640 1 -- 172.21.15.98:0/623982275 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c90011630 con 0x7f7ca0152ae0 2024-08-14T06:52:14.489 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.488+0000 7f7c8effd640 1 -- 172.21.15.98:0/623982275 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f7c900118a0 con 0x7f7ca0152ae0 2024-08-14T06:52:14.490 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.489+0000 7f7c8effd640 1 -- 172.21.15.98:0/623982275 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c90011cb0 con 0x7f7ca0152ae0 2024-08-14T06:52:14.490 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.489+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7ca00b4d40 con 0x7f7ca0152ae0 2024-08-14T06:52:14.490 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.490+0000 7f7c8effd640 1 -- 172.21.15.98:0/623982275 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f7c9001d800 con 0x7f7ca0152ae0 2024-08-14T06:52:14.491 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.490+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7ca00b5300 con 0x7f7ca0152ae0 2024-08-14T06:52:14.491 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.490+0000 7f7c8effd640 1 -- 172.21.15.98:0/623982275 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f7c9001dd20 con 0x7f7ca0152ae0 2024-08-14T06:52:14.496 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.495+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7ca006e300 con 0x7f7ca0152ae0 2024-08-14T06:52:14.500 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.500+0000 7f7c8effd640 1 -- 172.21.15.98:0/623982275 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f7ca006e300 con 0x7f7ca0152ae0 2024-08-14T06:52:14.505 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.492+0000 7ff0c92d7640 1 -- 172.21.15.98:0/3963056040 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_command([{prefix=config set, name=debug_mds}] v 0) -- 0x7ff09c0050e0 con 0x7ff0c41627f0 2024-08-14T06:52:14.512 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.511+0000 7ff0ab7fe640 1 -- 172.21.15.98:0/3963056040 <== mon.1 v2:172.21.15.191:3300/0 11 ==== mon_command_ack([{prefix=config set, name=debug_mds}]=0 v6)=0 v6) ==== 108+0+0 (secure 0 0 0) 0x7ff0b80035f0 con 0x7ff0c41627f0 2024-08-14T06:52:14.516 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.516+0000 7ff0c92d7640 1 -- 172.21.15.98:0/3963056040 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7ff0c41627f0 msgr2=0x7ff0c42053f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.516 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.516+0000 7ff0c92d7640 1 --2- 172.21.15.98:0/3963056040 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7ff0c41627f0 0x7ff0c42053f0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7ff0b800bed0 tx=0x7ff0b80095b0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.518+0000 7ff0c92d7640 1 -- 172.21.15.98:0/3963056040 shutdown_connections 2024-08-14T06:52:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.518+0000 7ff0c92d7640 1 --2- 172.21.15.98:0/3963056040 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7ff0c41627f0 0x7ff0c42053f0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.518+0000 7ff0c92d7640 1 --2- 172.21.15.98:0/3963056040 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7ff0c40710a0 0x7ff0c4157d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.518+0000 7ff0c92d7640 1 --2- 172.21.15.98:0/3963056040 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7ff0c406fd60 0x7ff0c41ff1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.518+0000 7ff0c92d7640 1 -- 172.21.15.98:0/3963056040 >> 172.21.15.98:0/3963056040 conn(0x7ff0c406d650 msgr2=0x7ff0c40b25a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.519 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.519+0000 7fdc85cf9640 1 -- 172.21.15.98:0/3091478896 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command([{prefix=config set, name=rados_mon_op_timeout}] v 0) -- 0x7fdc58005100 con 0x7fdc8015efb0 2024-08-14T06:52:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.520+0000 7ff0c92d7640 1 -- 172.21.15.98:0/3963056040 shutdown_connections 2024-08-14T06:52:14.520 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.520+0000 7ff0c92d7640 1 -- 172.21.15.98:0/3963056040 wait complete. 2024-08-14T06:52:14.532 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.530+0000 7f4f2d7fa640 1 -- 172.21.15.98:0/3923728278 <== mon.2 v2:172.21.15.191:3301/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4f28003d40 con 0x7f4f300cdf40 2024-08-14T06:52:14.532 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.532+0000 7f2c677fe640 1 -- 172.21.15.98:0/3265477866 <== mon.0 v2:172.21.15.98:3300/0 8 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2c70010450 con 0x7f2c801f45a0 2024-08-14T06:52:14.534 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.532+0000 7fdc67fff640 1 -- 172.21.15.98:0/3091478896 <== mon.2 v2:172.21.15.191:3301/0 9 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fdc78034bb0 con 0x7fdc8015efb0 2024-08-14T06:52:14.534 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.532+0000 7fdc67fff640 1 -- 172.21.15.98:0/3091478896 <== mon.2 v2:172.21.15.191:3301/0 10 ==== mon_command_ack([{prefix=config set, name=rados_mon_op_timeout}]=0 v7)=0 v7) ==== 123+0+0 (secure 0 0 0) 0x7fdc78004430 con 0x7fdc8015efb0 2024-08-14T06:52:14.539 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.538+0000 7f7c8effd640 1 -- 172.21.15.98:0/623982275 <== mon.0 v2:172.21.15.98:3300/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7c9001ca10 con 0x7f7ca0152ae0 2024-08-14T06:52:14.543 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.542+0000 7f0071ffb640 1 -- 172.21.15.98:0/3744203810 <== mon.0 v2:172.21.15.98:3300/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0074004430 con 0x7f007c0aea60 2024-08-14T06:52:14.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.548+0000 7fdc65ffb640 1 -- 172.21.15.98:0/3091478896 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdc8015efb0 msgr2=0x7fdc801f3ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.548+0000 7fdc65ffb640 1 --2- 172.21.15.98:0/3091478896 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdc8015efb0 0x7fdc801f3ee0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7fdc7802c380 tx=0x7fdc78004ab0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.548+0000 7fdc65ffb640 1 -- 172.21.15.98:0/3091478896 shutdown_connections 2024-08-14T06:52:14.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.548+0000 7fdc65ffb640 1 --2- 172.21.15.98:0/3091478896 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdc8015efb0 0x7fdc801f3ee0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.548+0000 7fdc65ffb640 1 --2- 172.21.15.98:0/3091478896 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fdc8015e660 0x7fdc80159af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.548+0000 7fdc65ffb640 1 --2- 172.21.15.98:0/3091478896 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fdc80143d70 0x7fdc800af450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.550 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.548+0000 7fdc65ffb640 1 -- 172.21.15.98:0/3091478896 >> 172.21.15.98:0/3091478896 conn(0x7fdc8006d650 msgr2=0x7fdc8013db80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.550 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.548+0000 7fdc65ffb640 1 -- 172.21.15.98:0/3091478896 shutdown_connections 2024-08-14T06:52:14.550 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.549+0000 7fdc65ffb640 1 -- 172.21.15.98:0/3091478896 wait complete. 2024-08-14T06:52:14.557 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.557+0000 7f00677fe640 1 -- 172.21.15.98:0/3744203810 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command([{prefix=config set, name=mds_debug_frag}] v 0) -- 0x7f00540052f0 con 0x7f007c0aea60 2024-08-14T06:52:14.570 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.569+0000 7f0071ffb640 1 -- 172.21.15.98:0/3744203810 <== mon.0 v2:172.21.15.98:3300/0 8 ==== mon_command_ack([{prefix=config set, name=mds_debug_frag}]=0 v8)=0 v8) ==== 115+0+0 (secure 0 0 0) 0x7f00740045e0 con 0x7f007c0aea60 2024-08-14T06:52:14.573 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.572+0000 7f008208d640 1 -- 172.21.15.98:0/3744203810 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f007c0aea60 msgr2=0x7f007c158fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.573 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.572+0000 7f008208d640 1 --2- 172.21.15.98:0/3744203810 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f007c0aea60 0x7f007c158fa0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f0074009cf0 tx=0x7f0074032f70 comp rx=0 tx=0).stop 2024-08-14T06:52:14.573 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.572+0000 7f008208d640 1 -- 172.21.15.98:0/3744203810 shutdown_connections 2024-08-14T06:52:14.574 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.572+0000 7f008208d640 1 --2- 172.21.15.98:0/3744203810 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f007c0cda80 0x7f007c15b8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.574 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.572+0000 7f008208d640 1 --2- 172.21.15.98:0/3744203810 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f007c0aea60 0x7f007c158fa0 secure :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f0074009cf0 tx=0x7f0074032f70 comp rx=0 tx=0).stop 2024-08-14T06:52:14.574 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.572+0000 7f008208d640 1 --2- 172.21.15.98:0/3744203810 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f007c06ed80 0x7f007c157170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.574 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.572+0000 7f008208d640 1 -- 172.21.15.98:0/3744203810 >> 172.21.15.98:0/3744203810 conn(0x7f007c0b1e60 msgr2=0x7f007c1578b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.574 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.573+0000 7f008208d640 1 -- 172.21.15.98:0/3744203810 shutdown_connections 2024-08-14T06:52:14.575 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.573+0000 7f008208d640 1 -- 172.21.15.98:0/3744203810 wait complete. 2024-08-14T06:52:14.578 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.577+0000 7f6939e09640 1 -- 172.21.15.98:0/2750998015 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f693408b1e0 msgr2=0x7f69340709b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.578 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.577+0000 7f6939e09640 1 --2- 172.21.15.98:0/2750998015 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f693408b1e0 0x7f69340709b0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f6924009800 tx=0x7f692402a3f0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.578 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.578+0000 7f6939e09640 1 -- 172.21.15.98:0/2750998015 shutdown_connections 2024-08-14T06:52:14.578 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.578+0000 7f6939e09640 1 --2- 172.21.15.98:0/2750998015 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f693408b1e0 0x7f69340709b0 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.579 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.578+0000 7f6939e09640 1 -- 172.21.15.98:0/2750998015 >> 172.21.15.98:0/2750998015 conn(0x7f69340cfcf0 msgr2=0x7f69340d0100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.582 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'rados_osd_op_timeout', '900'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.582 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_mds', '20'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.587 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.578+0000 7f6939e09640 1 -- 172.21.15.98:0/2750998015 shutdown_connections 2024-08-14T06:52:14.589 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.588+0000 7fc83a0bb640 1 -- 172.21.15.98:0/3968648850 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc8341665a0 msgr2=0x7fc8341669b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.589 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.588+0000 7fc83a0bb640 1 --2- 172.21.15.98:0/3968648850 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc8341665a0 0x7fc8341669b0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7fc824009830 tx=0x7fc82402a3f0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.590 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.588+0000 7fc83a0bb640 1 -- 172.21.15.98:0/3968648850 shutdown_connections 2024-08-14T06:52:14.590 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.588+0000 7fc83a0bb640 1 --2- 172.21.15.98:0/3968648850 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc8341665a0 0x7fc8341669b0 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.590 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.588+0000 7fc83a0bb640 1 --2- 172.21.15.98:0/3968648850 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc83406db60 0x7fc8340d3420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.590 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.588+0000 7fc83a0bb640 1 -- 172.21.15.98:0/3968648850 >> 172.21.15.98:0/3968648850 conn(0x7fc8340d4280 msgr2=0x7fc8340d4690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.592 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_mds_balancer', '20'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.592 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.578+0000 7f6939e09640 1 -- 172.21.15.98:0/2750998015 wait complete. 2024-08-14T06:52:14.592 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.585+0000 7f6939e09640 1 Processor -- start 2024-08-14T06:52:14.593 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.585+0000 7f6939e09640 1 -- start start 2024-08-14T06:52:14.593 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.591+0000 7fc83a0bb640 1 -- 172.21.15.98:0/3968648850 shutdown_connections 2024-08-14T06:52:14.593 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.591+0000 7fc83a0bb640 1 -- 172.21.15.98:0/3968648850 wait complete. 2024-08-14T06:52:14.596 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.592+0000 7f6939e09640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f693408a940 0x7f69342019c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.596 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.592+0000 7f6939e09640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f693408ad50 0x7f69340b4f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.596 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.592+0000 7f6932ffd640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f693408ad50 0x7f69340b4f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.596 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.592+0000 7f6932ffd640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f693408ad50 0x7f69340b4f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:40230/0 (socket says 172.21.15.98:40230) 2024-08-14T06:52:14.596 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.592+0000 7f6932ffd640 1 -- 172.21.15.98:0/4232177470 learned_addr learned my addr 172.21.15.98:0/4232177470 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.597 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.592+0000 7f6939e09640 1 --2- 172.21.15.98:0/4232177470 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f69340b2cd0 0x7f69340b54d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.597 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.592+0000 7f6939e09640 1 -- 172.21.15.98:0/4232177470 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f69340ce360 con 0x7f693408ad50 2024-08-14T06:52:14.597 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.592+0000 7f6939e09640 1 -- 172.21.15.98:0/4232177470 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f69340ce1e0 con 0x7f693408a940 2024-08-14T06:52:14.597 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.592+0000 7f6939e09640 1 -- 172.21.15.98:0/4232177470 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f69340cbe90 con 0x7f69340b2cd0 2024-08-14T06:52:14.597 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.593+0000 7f6932ffd640 1 -- 172.21.15.98:0/4232177470 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f69340b2cd0 msgr2=0x7f69340b54d0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-08-14T06:52:14.597 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.593+0000 7f6932ffd640 1 --2- 172.21.15.98:0/4232177470 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f69340b2cd0 0x7f69340b54d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.597 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.593+0000 7f6932ffd640 1 -- 172.21.15.98:0/4232177470 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f693408a940 msgr2=0x7f69342019c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-08-14T06:52:14.598 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.593+0000 7f6932ffd640 1 --2- 172.21.15.98:0/4232177470 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f693408a940 0x7f69342019c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.598 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.593+0000 7f6932ffd640 1 -- 172.21.15.98:0/4232177470 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6934205870 con 0x7f693408ad50 2024-08-14T06:52:14.598 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.593+0000 7f6932ffd640 1 --2- 172.21.15.98:0/4232177470 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f693408ad50 0x7f69340b4f90 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f692c00cc10 tx=0x7f692c00a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.598 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.595+0000 7f2c85326640 1 -- 172.21.15.98:0/3265477866 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command([{prefix=config set, name=debug_mds_balancer}] v 0) -- 0x7f2c580052f0 con 0x7f2c801f45a0 2024-08-14T06:52:14.599 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.595+0000 7f691bfff640 1 -- 172.21.15.98:0/4232177470 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f692c028070 con 0x7f693408ad50 2024-08-14T06:52:14.599 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.595+0000 7f6939e09640 1 -- 172.21.15.98:0/4232177470 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6934205b50 con 0x7f693408ad50 2024-08-14T06:52:14.599 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.595+0000 7f6939e09640 1 -- 172.21.15.98:0/4232177470 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6934206110 con 0x7f693408ad50 2024-08-14T06:52:14.599 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.597+0000 7f691bfff640 1 -- 172.21.15.98:0/4232177470 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f692c00ce50 con 0x7f693408ad50 2024-08-14T06:52:14.599 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.597+0000 7f6919ffb640 1 -- 172.21.15.98:0/4232177470 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f69340cc400 con 0x7f693408ad50 2024-08-14T06:52:14.600 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.599+0000 7fc83a0bb640 1 Processor -- start 2024-08-14T06:52:14.601 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.600+0000 7f691bfff640 1 -- 172.21.15.98:0/4232177470 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f692c00ad80 con 0x7f693408ad50 2024-08-14T06:52:14.601 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.600+0000 7f691bfff640 1 -- 172.21.15.98:0/4232177470 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f692c02b070 con 0x7f693408ad50 2024-08-14T06:52:14.601 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.600+0000 7f691bfff640 1 -- 172.21.15.98:0/4232177470 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f692c02d440 con 0x7f693408ad50 2024-08-14T06:52:14.602 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.602+0000 7fc83a0bb640 1 -- start start 2024-08-14T06:52:14.605 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.604+0000 7fc83a0bb640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc83406db60 0x7fc8341fa7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.605 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.604+0000 7fc83a0bb640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc8341665a0 0x7fc8340c8e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.605 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.604+0000 7fc83a0bb640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc834166ef0 0x7fc8340cde50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.605 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.604+0000 7fc83a0bb640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7fc83409d160 con 0x7fc8341665a0 2024-08-14T06:52:14.605 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.604+0000 7fc83a0bb640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7fc834167510 con 0x7fc83406db60 2024-08-14T06:52:14.605 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.604+0000 7fc833fff640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc834166ef0 0x7fc8340cde50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.606 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.604+0000 7fc833fff640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc834166ef0 0x7fc8340cde50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3301/0 says I am v2:172.21.15.98:45592/0 (socket says 172.21.15.98:45592) 2024-08-14T06:52:14.606 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.604+0000 7fc833fff640 1 -- 172.21.15.98:0/1538418903 learned_addr learned my addr 172.21.15.98:0/1538418903 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.606 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.605+0000 7fc83a0bb640 1 -- 172.21.15.98:0/1538418903 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7fc83409d2e0 con 0x7fc834166ef0 2024-08-14T06:52:14.607 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.605+0000 7fc833fff640 1 -- 172.21.15.98:0/1538418903 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc83406db60 msgr2=0x7fc8341fa7e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-08-14T06:52:14.607 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.605+0000 7fc833fff640 1 --2- 172.21.15.98:0/1538418903 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc83406db60 0x7fc8341fa7e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.607 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.605+0000 7fc833fff640 1 -- 172.21.15.98:0/1538418903 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc8341665a0 msgr2=0x7fc8340c8e00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-08-14T06:52:14.607 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.605+0000 7fc833fff640 1 --2- 172.21.15.98:0/1538418903 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc8341665a0 0x7fc8340c8e00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.607 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.605+0000 7fc833fff640 1 -- 172.21.15.98:0/1538418903 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc8340ccdd0 con 0x7fc834166ef0 2024-08-14T06:52:14.607 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.605+0000 7fc833fff640 1 --2- 172.21.15.98:0/1538418903 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc834166ef0 0x7fc8340cde50 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fc82c00ca60 tx=0x7fc82c00cf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.607 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.606+0000 7fc830ff9640 1 -- 172.21.15.98:0/1538418903 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc82c017070 con 0x7fc834166ef0 2024-08-14T06:52:14.607 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.606+0000 7fc83a0bb640 1 -- 172.21.15.98:0/1538418903 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc8340cd090 con 0x7fc834166ef0 2024-08-14T06:52:14.608 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.606+0000 7fc83a0bb640 1 -- 172.21.15.98:0/1538418903 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fc8340cd650 con 0x7fc834166ef0 2024-08-14T06:52:14.608 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.606+0000 7fc830ff9640 1 -- 172.21.15.98:0/1538418903 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc82c00a700 con 0x7fc834166ef0 2024-08-14T06:52:14.608 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.606+0000 7fc830ff9640 1 -- 172.21.15.98:0/1538418903 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc82c00ab50 con 0x7fc834166ef0 2024-08-14T06:52:14.608 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.606+0000 7fc830ff9640 1 -- 172.21.15.98:0/1538418903 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fc82c01a070 con 0x7fc834166ef0 2024-08-14T06:52:14.608 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.606+0000 7fc830ff9640 1 -- 172.21.15.98:0/1538418903 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fc82c01c840 con 0x7fc834166ef0 2024-08-14T06:52:14.611 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.608+0000 7f691bfff640 1 -- 172.21.15.98:0/4232177470 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f692c005070 con 0x7f693408ad50 2024-08-14T06:52:14.611 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.609+0000 7f59424fb640 1 -- 172.21.15.98:0/1054822429 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f593c15b400 msgr2=0x7f593c06e6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.611 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.609+0000 7f59424fb640 1 --2- 172.21.15.98:0/1054822429 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f593c15b400 0x7f593c06e6c0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f593400afd0 tx=0x7f593402b680 comp rx=0 tx=0).stop 2024-08-14T06:52:14.612 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.612+0000 7f2c677fe640 1 -- 172.21.15.98:0/3265477866 <== mon.0 v2:172.21.15.98:3300/0 9 ==== mon_command_ack([{prefix=config set, name=debug_mds_balancer}]=0 v9)=0 v9) ==== 117+0+0 (secure 0 0 0) 0x7f2c801f4ef0 con 0x7f2c801f45a0 2024-08-14T06:52:14.615 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.615+0000 7fc83a0bb640 1 -- 172.21.15.98:0/1538418903 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc818005290 con 0x7fc834166ef0 2024-08-14T06:52:14.617 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.616+0000 7f2c85326640 1 -- 172.21.15.98:0/3265477866 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2c801f45a0 msgr2=0x7f2c801f49b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.617 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.616+0000 7f2c85326640 1 --2- 172.21.15.98:0/3265477866 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2c801f45a0 0x7f2c801f49b0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f2c7000d920 tx=0x7f2c7000dde0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.617 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.616+0000 7f2c85326640 1 -- 172.21.15.98:0/3265477866 shutdown_connections 2024-08-14T06:52:14.617 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.616+0000 7f2c85326640 1 --2- 172.21.15.98:0/3265477866 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f2c8015c880 0x7f2c8015cc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.617 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.616+0000 7f2c85326640 1 --2- 172.21.15.98:0/3265477866 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2c801f45a0 0x7f2c801f49b0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.617 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.616+0000 7f2c85326640 1 --2- 172.21.15.98:0/3265477866 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f2c801f5260 0x7f2c8013cbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.617 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.616+0000 7f2c85326640 1 -- 172.21.15.98:0/3265477866 >> 172.21.15.98:0/3265477866 conn(0x7f2c8006d650 msgr2=0x7f2c800b0540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.618 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.617+0000 7f2c85326640 1 -- 172.21.15.98:0/3265477866 shutdown_connections 2024-08-14T06:52:14.618 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.617+0000 7f2c85326640 1 -- 172.21.15.98:0/3265477866 wait complete. 2024-08-14T06:52:14.619 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.617+0000 7fc830ff9640 1 -- 172.21.15.98:0/1538418903 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7fc82c021020 con 0x7fc834166ef0 2024-08-14T06:52:14.619 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.617+0000 7f59424fb640 1 -- 172.21.15.98:0/1054822429 shutdown_connections 2024-08-14T06:52:14.620 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.617+0000 7f59424fb640 1 --2- 172.21.15.98:0/1054822429 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f593c15b400 0x7f593c06e6c0 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.620 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.617+0000 7f59424fb640 1 --2- 172.21.15.98:0/1054822429 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f593c15aab0 0x7f593c15aec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.620 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.617+0000 7f59424fb640 1 -- 172.21.15.98:0/1054822429 >> 172.21.15.98:0/1054822429 conn(0x7f593c0b1e60 msgr2=0x7f593c0cfca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.620 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.617+0000 7f59424fb640 1 -- 172.21.15.98:0/1054822429 shutdown_connections 2024-08-14T06:52:14.620 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.617+0000 7f59424fb640 1 -- 172.21.15.98:0/1054822429 wait complete. 2024-08-14T06:52:14.621 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.618+0000 7f091b67d640 1 -- 172.21.15.98:0/4139543611 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f090c0e6ff0 msgr2=0x7f090c005dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.621 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.618+0000 7f091b67d640 1 --2- 172.21.15.98:0/4139543611 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f090c0e6ff0 0x7f090c005dd0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f0914052510 tx=0x7f091406c120 comp rx=0 tx=0).stop 2024-08-14T06:52:14.621 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.618+0000 7f091b67d640 1 -- 172.21.15.98:0/4139543611 shutdown_connections 2024-08-14T06:52:14.622 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.618+0000 7f091b67d640 1 --2- 172.21.15.98:0/4139543611 >> v2:172.21.15.191:3300/0 conn(0x7f090c0e7400 0x7f090c006310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.622 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.618+0000 7f091b67d640 1 --2- 172.21.15.98:0/4139543611 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f090c0e6ff0 0x7f090c005dd0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.622 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.618+0000 7f091b67d640 1 --2- 172.21.15.98:0/4139543611 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f090c052350 0x7f090c0e6ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.622 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.618+0000 7f091b67d640 1 -- 172.21.15.98:0/4139543611 >> 172.21.15.98:0/4139543611 conn(0x7f090c01d990 msgr2=0x7f090c039650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.622 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.618+0000 7f091b67d640 1 -- 172.21.15.98:0/4139543611 shutdown_connections 2024-08-14T06:52:14.623 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.619+0000 7f091b67d640 1 -- 172.21.15.98:0/4139543611 wait complete. 2024-08-14T06:52:14.623 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.621+0000 7f091b67d640 1 Processor -- start 2024-08-14T06:52:14.624 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.623+0000 7f3c20d6b640 1 -- 172.21.15.98:0/1170634549 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3c1c15c1b0 msgr2=0x7f3c1c0ce030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.624 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.623+0000 7f3c20d6b640 1 --2- 172.21.15.98:0/1170634549 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3c1c15c1b0 0x7f3c1c0ce030 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f3c10009860 tx=0x7f3c1002a460 comp rx=0 tx=0).stop 2024-08-14T06:52:14.625 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.623+0000 7f3c20d6b640 1 -- 172.21.15.98:0/1170634549 shutdown_connections 2024-08-14T06:52:14.625 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.623+0000 7f3c20d6b640 1 --2- 172.21.15.98:0/1170634549 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3c1c15c1b0 0x7f3c1c0ce030 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.625 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.623+0000 7f3c20d6b640 1 -- 172.21.15.98:0/1170634549 >> 172.21.15.98:0/1170634549 conn(0x7f3c1c0cfee0 msgr2=0x7f3c1c0d02f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.621+0000 7f091b67d640 1 -- start start 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f091b67d640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f090c052350 0x7f090c0d7770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f091b67d640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f090c0e7400 0x7f090c0dfa60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f091b67d640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f090c175c90 0x7f090c0e2480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f091b67d640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f090c0e3580 con 0x7f090c052350 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f091b67d640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f090c0e3400 con 0x7f090c175c90 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f091b67d640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f090c054710 con 0x7f090c0e7400 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f0919419640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f090c052350 0x7f090c0d7770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f0918c18640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f090c0e7400 0x7f090c0dfa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f0918c18640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f090c0e7400 0x7f090c0dfa60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3301/0 says I am v2:172.21.15.98:45616/0 (socket says 172.21.15.98:45616) 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f0918c18640 1 -- 172.21.15.98:0/1900960511 learned_addr learned my addr 172.21.15.98:0/1900960511 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.624+0000 7f0919c1a640 1 --2- 172.21.15.98:0/1900960511 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f090c175c90 0x7f090c0e2480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.627 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.625+0000 7f0918c18640 1 -- 172.21.15.98:0/1900960511 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f090c175c90 msgr2=0x7f090c0e2480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.628 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.625+0000 7f59424fb640 1 Processor -- start 2024-08-14T06:52:14.628 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.625+0000 7f59424fb640 1 -- start start 2024-08-14T06:52:14.629 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.625+0000 7f0918c18640 1 --2- 172.21.15.98:0/1900960511 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f090c175c90 0x7f090c0e2480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.629 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.625+0000 7f0918c18640 1 -- 172.21.15.98:0/1900960511 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f090c052350 msgr2=0x7f090c0d7770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.629 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.625+0000 7f0918c18640 1 --2- 172.21.15.98:0/1900960511 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f090c052350 0x7f090c0d7770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.629 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.625+0000 7f0918c18640 1 -- 172.21.15.98:0/1900960511 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f090c0e1270 con 0x7f090c0e7400 2024-08-14T06:52:14.629 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.625+0000 7f3c20d6b640 1 -- 172.21.15.98:0/1170634549 shutdown_connections 2024-08-14T06:52:14.630 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.625+0000 7f0919419640 1 --2- 172.21.15.98:0/1900960511 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f090c052350 0x7f090c0d7770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:14.630 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.628+0000 7f0918c18640 1 --2- 172.21.15.98:0/1900960511 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f090c0e7400 0x7f090c0dfa60 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f09140519c0 tx=0x7f09140b8d80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.632 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.631+0000 7f3c20d6b640 1 -- 172.21.15.98:0/1170634549 wait complete. 2024-08-14T06:52:14.634 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.632+0000 7f0909ffb640 1 -- 172.21.15.98:0/1900960511 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09140c4050 con 0x7f090c0e7400 2024-08-14T06:52:14.634 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.632+0000 7f091b67d640 1 -- 172.21.15.98:0/1900960511 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f090c1788c0 con 0x7f090c0e7400 2024-08-14T06:52:14.635 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.632+0000 7f091b67d640 1 -- 172.21.15.98:0/1900960511 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f090c178e80 con 0x7f090c0e7400 2024-08-14T06:52:14.635 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.634+0000 7f0909ffb640 1 -- 172.21.15.98:0/1900960511 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f09140c5040 con 0x7f090c0e7400 2024-08-14T06:52:14.635 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.634+0000 7f0909ffb640 1 -- 172.21.15.98:0/1900960511 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09140c9870 con 0x7f090c0e7400 2024-08-14T06:52:14.635 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.634+0000 7f3c20d6b640 1 Processor -- start 2024-08-14T06:52:14.636 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.635+0000 7f0909ffb640 1 -- 172.21.15.98:0/1900960511 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f09140d2020 con 0x7f090c0e7400 2024-08-14T06:52:14.636 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.635+0000 7f3c20d6b640 1 -- start start 2024-08-14T06:52:14.636 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.635+0000 7f0909ffb640 1 -- 172.21.15.98:0/1900960511 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f09140cb450 con 0x7f090c0e7400 2024-08-14T06:52:14.636 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.636+0000 7f08ff7fe640 1 -- 172.21.15.98:0/1900960511 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f08f4005290 con 0x7f090c0e7400 2024-08-14T06:52:14.638 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.637+0000 7f59424fb640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f593c0cad00 0x7f593c1f9400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.638 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.637+0000 7f3c20d6b640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3c1c072ca0 0x7f3c1c06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.638 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.637+0000 7f3c20d6b640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f3c1c15c1b0 0x7f3c1c14f470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.638 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.637+0000 7f3c20d6b640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3c1c1fe210 0x7f3c1c1fd610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.639 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.637+0000 7f59424fb640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f593c15aab0 0x7f593c1431b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.639 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.637+0000 7f3c20d6b640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f3c1c070ea0 con 0x7f3c1c072ca0 2024-08-14T06:52:14.639 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.637+0000 7f3c20d6b640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f3c1c070d20 con 0x7f3c1c15c1b0 2024-08-14T06:52:14.639 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.637+0000 7f3c20d6b640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f3c1c071020 con 0x7f3c1c1fe210 2024-08-14T06:52:14.640 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.638+0000 7f59414f9640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f593c0cad00 0x7f593c1f9400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.640 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.638+0000 7f59414f9640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f593c0cad00 0x7f593c1f9400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3301/0 says I am v2:172.21.15.98:45632/0 (socket says 172.21.15.98:45632) 2024-08-14T06:52:14.640 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.638+0000 7f5940cf8640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f593c15aab0 0x7f593c1431b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.640 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.638+0000 7f5940cf8640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f593c15aab0 0x7f593c1431b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3300/0 says I am v2:172.21.15.98:52928/0 (socket says 172.21.15.98:52928) 2024-08-14T06:52:14.640 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.638+0000 7f59414f9640 1 -- 172.21.15.98:0/3713758045 learned_addr learned my addr 172.21.15.98:0/3713758045 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.640 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.638+0000 7f59424fb640 1 --2- 172.21.15.98:0/3713758045 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f593c15b400 0x7f593c145a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.642 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.638+0000 7f3c1ad9d640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3c1c1fe210 0x7f3c1c1fd610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.642 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.638+0000 7f3c1ad9d640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3c1c1fe210 0x7f3c1c1fd610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3301/0 says I am v2:172.21.15.98:45646/0 (socket says 172.21.15.98:45646) 2024-08-14T06:52:14.642 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.638+0000 7f3c1ad9d640 1 -- 172.21.15.98:0/125781923 learned_addr learned my addr 172.21.15.98:0/125781923 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.642 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.638+0000 7f3c1a59c640 1 --2- 172.21.15.98:0/125781923 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3c1c072ca0 0x7f3c1c06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.642 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.639+0000 7f3c1ad9d640 1 -- 172.21.15.98:0/125781923 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f3c1c15c1b0 msgr2=0x7f3c1c14f470 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-08-14T06:52:14.642 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.639+0000 7f3c1ad9d640 1 --2- 172.21.15.98:0/125781923 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f3c1c15c1b0 0x7f3c1c14f470 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.642 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.639+0000 7f3c1ad9d640 1 -- 172.21.15.98:0/125781923 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3c1c072ca0 msgr2=0x7f3c1c06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.642 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.639+0000 7f3c1ad9d640 1 --2- 172.21.15.98:0/125781923 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3c1c072ca0 0x7f3c1c06e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.642 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.639+0000 7f3c1ad9d640 1 -- 172.21.15.98:0/125781923 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3c1c14f9b0 con 0x7f3c1c1fe210 2024-08-14T06:52:14.643 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.639+0000 7f9036fa3640 1 -- 172.21.15.98:0/343108560 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f90280df7f0 msgr2=0x7f9028007c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.643 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.639+0000 7f9036fa3640 1 --2- 172.21.15.98:0/343108560 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f90280df7f0 0x7f9028007c50 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f90280e30b0 tx=0x7f902c02a460 comp rx=0 tx=0).stop 2024-08-14T06:52:14.643 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.639+0000 7f3c1a59c640 1 --2- 172.21.15.98:0/125781923 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3c1c072ca0 0x7f3c1c06e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:14.644 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f5941cfa640 1 --2- 172.21.15.98:0/3713758045 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f593c15b400 0x7f593c145a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.644 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f593c06ed80 con 0x7f593c15b400 2024-08-14T06:52:14.644 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f593c06ec00 con 0x7f593c15aab0 2024-08-14T06:52:14.644 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f593c06ef00 con 0x7f593c0cad00 2024-08-14T06:52:14.645 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f9036fa3640 1 -- 172.21.15.98:0/343108560 shutdown_connections 2024-08-14T06:52:14.645 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f9036fa3640 1 --2- 172.21.15.98:0/343108560 >> v2:172.21.15.191:3300/0 conn(0x7f9028051990 0x7f9028051da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.645 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f9036fa3640 1 --2- 172.21.15.98:0/343108560 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f90280df7f0 0x7f9028007c50 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.645 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f9036fa3640 1 -- 172.21.15.98:0/343108560 >> 172.21.15.98:0/343108560 conn(0x7f902801e250 msgr2=0x7f9028039e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.645 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f5940cf8640 1 -- 172.21.15.98:0/3713758045 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f593c0cad00 msgr2=0x7f593c1f9400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.645 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f5940cf8640 1 --2- 172.21.15.98:0/3713758045 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f593c0cad00 0x7f593c1f9400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.645 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f5940cf8640 1 -- 172.21.15.98:0/3713758045 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f593c15b400 msgr2=0x7f593c145a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.645 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f5940cf8640 1 --2- 172.21.15.98:0/3713758045 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f593c15b400 0x7f593c145a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.646 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f5940cf8640 1 -- 172.21.15.98:0/3713758045 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f593c1452e0 con 0x7f593c15aab0 2024-08-14T06:52:14.646 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.641+0000 7f9036fa3640 1 -- 172.21.15.98:0/343108560 shutdown_connections 2024-08-14T06:52:14.646 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.641+0000 7f9036fa3640 1 -- 172.21.15.98:0/343108560 wait complete. 2024-08-14T06:52:14.647 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.640+0000 7f3c1ad9d640 1 --2- 172.21.15.98:0/125781923 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3c1c1fe210 0x7f3c1c1fd610 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f3c1400b9e0 tx=0x7f3c1400bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.647 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.642+0000 7f3c0affd640 1 -- 172.21.15.98:0/125781923 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c14019070 con 0x7f3c1c1fe210 2024-08-14T06:52:14.647 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.643+0000 7f3c20d6b640 1 -- 172.21.15.98:0/125781923 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3c1c15aec0 con 0x7f3c1c1fe210 2024-08-14T06:52:14.647 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.643+0000 7f3c20d6b640 1 -- 172.21.15.98:0/125781923 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f3c1c15b480 con 0x7f3c1c1fe210 2024-08-14T06:52:14.647 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.644+0000 7f9036fa3640 1 Processor -- start 2024-08-14T06:52:14.647 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.644+0000 7f9036fa3640 1 -- start start 2024-08-14T06:52:14.648 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.644+0000 7f3c0affd640 1 -- 172.21.15.98:0/125781923 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f3c1400cc10 con 0x7f3c1c1fe210 2024-08-14T06:52:14.648 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.645+0000 7f0909ffb640 1 -- 172.21.15.98:0/1900960511 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f09140c9a10 con 0x7f090c0e7400 2024-08-14T06:52:14.648 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.645+0000 7f3c08ff9640 1 -- 172.21.15.98:0/125781923 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3c00005290 con 0x7f3c1c1fe210 2024-08-14T06:52:14.650 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.648+0000 7f5940cf8640 1 --2- 172.21.15.98:0/3713758045 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f593c15aab0 0x7f593c1431b0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f593c06ff30 tx=0x7f592c00cd50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.650 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.649+0000 7f0575eb6640 1 -- 172.21.15.98:0/3449453625 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f05700aec30 msgr2=0x7f0570161ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.650 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.649+0000 7f0575eb6640 1 --2- 172.21.15.98:0/3449453625 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f05700aec30 0x7f0570161ed0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f0564005fd0 tx=0x7f056402a560 comp rx=0 tx=0).stop 2024-08-14T06:52:14.650 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.649+0000 7f0575eb6640 1 -- 172.21.15.98:0/3449453625 shutdown_connections 2024-08-14T06:52:14.651 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.649+0000 7f0575eb6640 1 --2- 172.21.15.98:0/3449453625 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f0570162410 0x7f0570162820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.651 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.649+0000 7f0575eb6640 1 --2- 172.21.15.98:0/3449453625 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f05700aec30 0x7f0570161ed0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.651 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.649+0000 7f0575eb6640 1 -- 172.21.15.98:0/3449453625 >> 172.21.15.98:0/3449453625 conn(0x7f05700b1e60 msgr2=0x7f05700cfca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.652 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.650+0000 7f5931ffb640 1 -- 172.21.15.98:0/3713758045 <== mon.1 v2:172.21.15.191:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f592c007bb0 con 0x7f593c15aab0 2024-08-14T06:52:14.652 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.650+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f593c1436f0 con 0x7f593c15aab0 2024-08-14T06:52:14.652 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.650+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f593c1f9f80 con 0x7f593c15aab0 2024-08-14T06:52:14.652 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.650+0000 7f5931ffb640 1 -- 172.21.15.98:0/3713758045 <== mon.1 v2:172.21.15.191:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f592c007e40 con 0x7f593c15aab0 2024-08-14T06:52:14.652 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.650+0000 7f5931ffb640 1 -- 172.21.15.98:0/3713758045 <== mon.1 v2:172.21.15.191:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f592c00f6b0 con 0x7f593c15aab0 2024-08-14T06:52:14.652 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.651+0000 7f5931ffb640 1 -- 172.21.15.98:0/3713758045 <== mon.1 v2:172.21.15.191:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f592c00f910 con 0x7f593c15aab0 2024-08-14T06:52:14.652 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.651+0000 7f5931ffb640 1 -- 172.21.15.98:0/3713758045 <== mon.1 v2:172.21.15.191:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f592c0213f0 con 0x7f593c15aab0 2024-08-14T06:52:14.652 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.650+0000 7f3c0affd640 1 -- 172.21.15.98:0/125781923 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c140146a0 con 0x7f3c1c1fe210 2024-08-14T06:52:14.653 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.650+0000 7f3c0affd640 1 -- 172.21.15.98:0/125781923 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f3c1401c070 con 0x7f3c1c1fe210 2024-08-14T06:52:14.653 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.650+0000 7f3c0affd640 1 -- 172.21.15.98:0/125781923 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f3c1401eab0 con 0x7f3c1c1fe210 2024-08-14T06:52:14.653 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.652+0000 7f9036fa3640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9028006810 0x7f90280da620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.653 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.652+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f593c15aec0 con 0x7f593c15aab0 2024-08-14T06:52:14.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.654+0000 7f0575eb6640 1 -- 172.21.15.98:0/3449453625 shutdown_connections 2024-08-14T06:52:14.655 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.654+0000 7f3c0affd640 1 -- 172.21.15.98:0/125781923 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f3c14023070 con 0x7f3c1c1fe210 2024-08-14T06:52:14.655 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.655+0000 7f9036fa3640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9028051990 0x7f902801ecc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.656 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_ms', '2'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.657 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.655+0000 7f9035fa1640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9028006810 0x7f90280da620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.657 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.655+0000 7f9035fa1640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9028006810 0x7f90280da620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3300/0 says I am v2:172.21.15.98:52938/0 (socket says 172.21.15.98:52938) 2024-08-14T06:52:14.657 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.655+0000 7f9035fa1640 1 -- 172.21.15.98:0/3206283197 learned_addr learned my addr 172.21.15.98:0/3206283197 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.657 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.655+0000 7f9036fa3640 1 --2- 172.21.15.98:0/3206283197 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f90280df7f0 0x7f902804fde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.657 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.655+0000 7f90357a0640 1 --2- 172.21.15.98:0/3206283197 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9028051990 0x7f902801ecc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.658 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.656+0000 7f9036fa3640 1 -- 172.21.15.98:0/3206283197 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f9028054890 con 0x7f9028051990 2024-08-14T06:52:14.658 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.656+0000 7f9036fa3640 1 -- 172.21.15.98:0/3206283197 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f9028054710 con 0x7f9028006810 2024-08-14T06:52:14.658 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.656+0000 7f4f36c09640 1 -- 172.21.15.98:0/3923728278 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command([{prefix=config set, name=debug_ms}] v 0) -- 0x7f4f10005370 con 0x7f4f300cdf40 2024-08-14T06:52:14.658 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.656+0000 7f90367a2640 1 --2- 172.21.15.98:0/3206283197 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f90280df7f0 0x7f902804fde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.659 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.656+0000 7f9036fa3640 1 -- 172.21.15.98:0/3206283197 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f9028054a10 con 0x7f90280df7f0 2024-08-14T06:52:14.659 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.657+0000 7f5931ffb640 1 -- 172.21.15.98:0/3713758045 <== mon.1 v2:172.21.15.191:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f592c014030 con 0x7f593c15aab0 2024-08-14T06:52:14.659 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.657+0000 7f90367a2640 1 -- 172.21.15.98:0/3206283197 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9028006810 msgr2=0x7f90280da620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.659 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.657+0000 7f90367a2640 1 --2- 172.21.15.98:0/3206283197 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9028006810 0x7f90280da620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.659 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.657+0000 7f90367a2640 1 -- 172.21.15.98:0/3206283197 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9028051990 msgr2=0x7f902801ecc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.659 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.657+0000 7f90367a2640 1 --2- 172.21.15.98:0/3206283197 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9028051990 0x7f902801ecc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.659 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.657+0000 7f90367a2640 1 -- 172.21.15.98:0/3206283197 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9028021860 con 0x7f90280df7f0 2024-08-14T06:52:14.660 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.656+0000 7f0575eb6640 1 -- 172.21.15.98:0/3449453625 wait complete. 2024-08-14T06:52:14.660 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.658+0000 7f90367a2640 1 --2- 172.21.15.98:0/3206283197 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f90280df7f0 0x7f902804fde0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f9030052510 tx=0x7f903006acb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.660 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.659+0000 7f90267fc640 1 -- 172.21.15.98:0/3206283197 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f90300945d0 con 0x7f90280df7f0 2024-08-14T06:52:14.661 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.659+0000 7f90267fc640 1 -- 172.21.15.98:0/3206283197 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9030094860 con 0x7f90280df7f0 2024-08-14T06:52:14.661 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.659+0000 7f90267fc640 1 -- 172.21.15.98:0/3206283197 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9030094cb0 con 0x7f90280df7f0 2024-08-14T06:52:14.661 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.659+0000 7f0575eb6640 1 Processor -- start 2024-08-14T06:52:14.661 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.659+0000 7f9036fa3640 1 -- 172.21.15.98:0/3206283197 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9028021b40 con 0x7f90280df7f0 2024-08-14T06:52:14.662 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.660+0000 7f90267fc640 1 -- 172.21.15.98:0/3206283197 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f9030095800 con 0x7f90280df7f0 2024-08-14T06:52:14.662 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.660+0000 7f9036fa3640 1 -- 172.21.15.98:0/3206283197 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f9028022100 con 0x7f90280df7f0 2024-08-14T06:52:14.662 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.662+0000 7f90267fc640 1 -- 172.21.15.98:0/3206283197 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f9030094e50 con 0x7f90280df7f0 2024-08-14T06:52:14.663 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.662+0000 7f9013fff640 1 -- 172.21.15.98:0/3206283197 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9028007450 con 0x7f90280df7f0 2024-08-14T06:52:14.665 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.664+0000 7f0575eb6640 1 -- start start 2024-08-14T06:52:14.666 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.665+0000 7f90267fc640 1 -- 172.21.15.98:0/3206283197 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f90280522e0 con 0x7f90280df7f0 2024-08-14T06:52:14.667 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.666+0000 7f0575eb6640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f05700aec30 0x7f05700cc8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.667 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.666+0000 7f0575eb6640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f05700cda80 0x7f05701fc900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.667 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.666+0000 7f0575eb6640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f0570162410 0x7f05701fce40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.667 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.666+0000 7f4f2d7fa640 1 -- 172.21.15.98:0/3923728278 <== mon.2 v2:172.21.15.191:3301/0 8 ==== mon_command_ack([{prefix=config set, name=debug_ms}]=0 v10)=0 v10) ==== 106+0+0 (secure 0 0 0) 0x7f4f28033c60 con 0x7f4f300cdf40 2024-08-14T06:52:14.668 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.667+0000 7f0575eb6640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f0570154a10 con 0x7f05700aec30 2024-08-14T06:52:14.668 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.667+0000 7f0575eb6640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f0570154890 con 0x7f05700cda80 2024-08-14T06:52:14.668 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.667+0000 7f0575eb6640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f0570154b90 con 0x7f0570162410 2024-08-14T06:52:14.669 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f4f1affd640 1 -- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f4f300cdf40 msgr2=0x7f4f300c9580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.669 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f4f1affd640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f4f300cdf40 0x7f4f300c9580 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f4f300ca4e0 tx=0x7f4f28033640 comp rx=0 tx=0).stop 2024-08-14T06:52:14.669 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f4f1affd640 1 -- 172.21.15.98:0/3923728278 shutdown_connections 2024-08-14T06:52:14.669 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f4f1affd640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f4f300cdf40 0x7f4f300c9580 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.669 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f4f1affd640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f4f3008b630 0x7f4f300c9040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.669 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f4f1affd640 1 --2- 172.21.15.98:0/3923728278 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f4f30070b80 0x7f4f300b2e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.669 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f4f1affd640 1 -- 172.21.15.98:0/3923728278 >> 172.21.15.98:0/3923728278 conn(0x7f4f300cfee0 msgr2=0x7f4f3006fe30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.670 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f4f1affd640 1 -- 172.21.15.98:0/3923728278 shutdown_connections 2024-08-14T06:52:14.670 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f4f1affd640 1 -- 172.21.15.98:0/3923728278 wait complete. 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.667+0000 7f0574eb4640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f05700aec30 0x7f05700cc8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.667+0000 7f0574eb4640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f05700aec30 0x7f05700cc8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:40318/0 (socket says 172.21.15.98:40318) 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.667+0000 7f0574eb4640 1 -- 172.21.15.98:0/1381416276 learned_addr learned my addr 172.21.15.98:0/1381416276 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f056ffff640 1 --2- 172.21.15.98:0/1381416276 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f05700cda80 0x7f05701fc900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f05756b5640 1 --2- 172.21.15.98:0/1381416276 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f0570162410 0x7f05701fce40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f0574eb4640 1 -- 172.21.15.98:0/1381416276 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f0570162410 msgr2=0x7f05701fce40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f0574eb4640 1 --2- 172.21.15.98:0/1381416276 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f0570162410 0x7f05701fce40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f0574eb4640 1 -- 172.21.15.98:0/1381416276 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f05700cda80 msgr2=0x7f05701fc900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f0574eb4640 1 --2- 172.21.15.98:0/1381416276 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f05700cda80 0x7f05701fc900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.668+0000 7f0574eb4640 1 -- 172.21.15.98:0/1381416276 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f05701fd650 con 0x7f05700aec30 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.669+0000 7f0574eb4640 1 --2- 172.21.15.98:0/1381416276 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f05700aec30 0x7f05700cc8a0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f056800c730 tx=0x7f056800cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.672 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.671+0000 7f056d7fa640 1 -- 172.21.15.98:0/1381416276 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0568003b70 con 0x7f05700aec30 2024-08-14T06:52:14.673 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.671+0000 7f0575eb6640 1 -- 172.21.15.98:0/1381416276 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f05701f9560 con 0x7f05700aec30 2024-08-14T06:52:14.673 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.671+0000 7f0575eb6640 1 -- 172.21.15.98:0/1381416276 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f05701f9b20 con 0x7f05700aec30 2024-08-14T06:52:14.673 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.671+0000 7f056d7fa640 1 -- 172.21.15.98:0/1381416276 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0568003d10 con 0x7f05700aec30 2024-08-14T06:52:14.673 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.671+0000 7f056d7fa640 1 -- 172.21.15.98:0/1381416276 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f056800aa50 con 0x7f05700aec30 2024-08-14T06:52:14.676 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.676+0000 7f056d7fa640 1 -- 172.21.15.98:0/1381416276 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f056800acb0 con 0x7f05700aec30 2024-08-14T06:52:14.676 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.674+0000 7f4512d38640 1 -- 172.21.15.98:0/105853723 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f450c073f80 msgr2=0x7f450c0caf40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.676 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.674+0000 7f4512d38640 1 --2- 172.21.15.98:0/105853723 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f450c073f80 0x7f450c0caf40 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f450c0716a0 tx=0x7f44fc02a420 comp rx=0 tx=0).stop 2024-08-14T06:52:14.677 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.676+0000 7f056d7fa640 1 -- 172.21.15.98:0/1381416276 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f0568012740 con 0x7f05700aec30 2024-08-14T06:52:14.682 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.678+0000 7f0575eb6640 1 -- 172.21.15.98:0/1381416276 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0550005290 con 0x7f05700aec30 2024-08-14T06:52:14.683 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.682+0000 7f4512d38640 1 -- 172.21.15.98:0/105853723 shutdown_connections 2024-08-14T06:52:14.683 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.682+0000 7f4512d38640 1 --2- 172.21.15.98:0/105853723 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f450c073f80 0x7f450c0caf40 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.683 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.682+0000 7f4512d38640 1 --2- 172.21.15.98:0/105853723 >> v2:172.21.15.191:3300/0 conn(0x7f450c072e80 0x7f450c0cf980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.683 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.682+0000 7f4512d38640 1 -- 172.21.15.98:0/105853723 >> 172.21.15.98:0/105853723 conn(0x7f450c0cfee0 msgr2=0x7f450c0d02f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.687 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_debug_frag', 'True'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.687 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_debug_scatterstat', 'True'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.687 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.686+0000 7f056d7fa640 1 -- 172.21.15.98:0/1381416276 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f05680050d0 con 0x7f05700aec30 2024-08-14T06:52:14.691 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.688+0000 7f4512d38640 1 -- 172.21.15.98:0/105853723 shutdown_connections 2024-08-14T06:52:14.692 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.692+0000 7f4512d38640 1 -- 172.21.15.98:0/105853723 wait complete. 2024-08-14T06:52:14.695 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.695+0000 7f4512d38640 1 Processor -- start 2024-08-14T06:52:14.695 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.695+0000 7f4512d38640 1 -- start start 2024-08-14T06:52:14.701 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.701+0000 7f4512d38640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f450c06fdf0 0x7f450c0b2d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.703 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.701+0000 7f4510ad4640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f450c06fdf0 0x7f450c0b2d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.703 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.701+0000 7f4510ad4640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f450c06fdf0 0x7f450c0b2d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:40322/0 (socket says 172.21.15.98:40322) 2024-08-14T06:52:14.703 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.702+0000 7f4510ad4640 1 -- 172.21.15.98:0/4212822085 learned_addr learned my addr 172.21.15.98:0/4212822085 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:14.703 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.702+0000 7f4512d38640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f450c072e80 0x7f450c152e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.703 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.702+0000 7f4512d38640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f450c073f80 0x7f450c153cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:14.703 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.702+0000 7f450bfff640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f450c072e80 0x7f450c152e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.703 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.702+0000 7f45112d5640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f450c073f80 0x7f450c153cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:14.703 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.702+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f450c0ce490 con 0x7f450c06fdf0 2024-08-14T06:52:14.703 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.702+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f450c0ce310 con 0x7f450c073f80 2024-08-14T06:52:14.703 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.702+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f450c0ce610 con 0x7f450c072e80 2024-08-14T06:52:14.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.706+0000 7f4510ad4640 1 -- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f450c072e80 msgr2=0x7f450c152e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.706+0000 7f4510ad4640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f450c072e80 0x7f450c152e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.706+0000 7f4510ad4640 1 -- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f450c073f80 msgr2=0x7f450c153cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.706+0000 7f4510ad4640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f450c073f80 0x7f450c153cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.706+0000 7f4510ad4640 1 -- 172.21.15.98:0/4212822085 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f450c1f9b90 con 0x7f450c06fdf0 2024-08-14T06:52:14.708 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.707+0000 7f4510ad4640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f450c06fdf0 0x7f450c0b2d10 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f450000b420 tx=0x7f450000b8e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:14.708 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.707+0000 7f45097fa640 1 -- 172.21.15.98:0/4212822085 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4500011830 con 0x7f450c06fdf0 2024-08-14T06:52:14.708 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.707+0000 7f45097fa640 1 -- 172.21.15.98:0/4212822085 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4500016070 con 0x7f450c06fdf0 2024-08-14T06:52:14.709 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.709+0000 7f45097fa640 1 -- 172.21.15.98:0/4212822085 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4500011d80 con 0x7f450c06fdf0 2024-08-14T06:52:14.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.710+0000 7f450bfff640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f450c072e80 0x7f450c152e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-08-14T06:52:14.714 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.713+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f450c0b3680 con 0x7f450c06fdf0 2024-08-14T06:52:14.715 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.713+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f450c1fa130 con 0x7f450c06fdf0 2024-08-14T06:52:14.715 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.714+0000 7f45097fa640 1 -- 172.21.15.98:0/4212822085 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f450001b460 con 0x7f450c06fdf0 2024-08-14T06:52:14.715 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.715+0000 7f45097fa640 1 -- 172.21.15.98:0/4212822085 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f450001a820 con 0x7f450c06fdf0 2024-08-14T06:52:14.722 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.715+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f450c0cf980 con 0x7f450c06fdf0 2024-08-14T06:52:14.723 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.723+0000 7f45097fa640 1 -- 172.21.15.98:0/4212822085 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f450001a3c0 con 0x7f450c06fdf0 2024-08-14T06:52:14.723 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.723+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command([{prefix=config set, name=mds_debug_scatterstat}] v 0) -- 0x7f7ca00b13e0 con 0x7f7ca0152ae0 2024-08-14T06:52:14.731 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.730+0000 7f7c8effd640 1 -- 172.21.15.98:0/623982275 <== mon.0 v2:172.21.15.98:3300/0 8 ==== mon_command_ack([{prefix=config set, name=mds_debug_scatterstat}]=0 v11)=0 v11) ==== 122+0+0 (secure 0 0 0) 0x7f7c90022020 con 0x7f7ca0152ae0 2024-08-14T06:52:14.732 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.731+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7ca0152ae0 msgr2=0x7f7ca0149730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.732 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.731+0000 7f7c9f59e640 1 --2- 172.21.15.98:0/623982275 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7ca0152ae0 0x7f7ca0149730 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f7ca00b27f0 tx=0x7f7c900016b0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.733 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.733+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 shutdown_connections 2024-08-14T06:52:14.733 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.733+0000 7f7c9f59e640 1 --2- 172.21.15.98:0/623982275 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7ca0152ae0 0x7f7ca0149730 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.733 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.733+0000 7f7c9f59e640 1 --2- 172.21.15.98:0/623982275 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7ca02017b0 0x7f7ca01525a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.734 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.733+0000 7f7c9f59e640 1 --2- 172.21.15.98:0/623982275 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7ca00b4860 0x7f7ca00cd0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.734 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.733+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 >> 172.21.15.98:0/623982275 conn(0x7f7ca00b1e60 msgr2=0x7f7ca0154b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.734 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.733+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 shutdown_connections 2024-08-14T06:52:14.734 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.733+0000 7f7c9f59e640 1 -- 172.21.15.98:0/623982275 wait complete. 2024-08-14T06:52:14.753 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_op_complaint_time', '180'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.756 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.755+0000 7f6919ffb640 1 -- 172.21.15.98:0/4232177470 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command([{prefix=config set, name=mon_op_complaint_time}] v 0) -- 0x7f69340b5c70 con 0x7f693408ad50 2024-08-14T06:52:14.760 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.760+0000 7fc83a0bb640 1 -- 172.21.15.98:0/1538418903 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command([{prefix=config set, name=mds_op_complaint_time}] v 0) -- 0x7fc818005580 con 0x7fc834166ef0 2024-08-14T06:52:14.762 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.762+0000 7f691bfff640 1 -- 172.21.15.98:0/4232177470 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_op_complaint_time}]=0 v12)=0 v12) ==== 121+0+0 (secure 0 0 0) 0x7f692c0057a0 con 0x7f693408ad50 2024-08-14T06:52:14.764 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.763+0000 7f6919ffb640 1 -- 172.21.15.98:0/4232177470 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f693408ad50 msgr2=0x7f69340b4f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.764 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.763+0000 7f6919ffb640 1 --2- 172.21.15.98:0/4232177470 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f693408ad50 0x7f69340b4f90 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f692c00cc10 tx=0x7f692c00a430 comp rx=0 tx=0).stop 2024-08-14T06:52:14.764 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.763+0000 7f6919ffb640 1 -- 172.21.15.98:0/4232177470 shutdown_connections 2024-08-14T06:52:14.764 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.763+0000 7f6919ffb640 1 --2- 172.21.15.98:0/4232177470 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f69340b2cd0 0x7f69340b54d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.764 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.763+0000 7f6919ffb640 1 --2- 172.21.15.98:0/4232177470 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f693408ad50 0x7f69340b4f90 secure :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f692c00cc10 tx=0x7f692c00a430 comp rx=0 tx=0).stop 2024-08-14T06:52:14.764 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.763+0000 7f6919ffb640 1 --2- 172.21.15.98:0/4232177470 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f693408a940 0x7f69342019c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.764 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.763+0000 7f6919ffb640 1 -- 172.21.15.98:0/4232177470 >> 172.21.15.98:0/4232177470 conn(0x7f69340cfcf0 msgr2=0x7f69340c3820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.765 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.764+0000 7f6919ffb640 1 -- 172.21.15.98:0/4232177470 shutdown_connections 2024-08-14T06:52:14.765 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.764+0000 7f6919ffb640 1 -- 172.21.15.98:0/4232177470 wait complete. 2024-08-14T06:52:14.768 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.767+0000 7fc830ff9640 1 -- 172.21.15.98:0/1538418903 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{prefix=config set, name=mds_op_complaint_time}]=0 v13)=0 v13) ==== 121+0+0 (secure 0 0 0) 0x7fc82c00add0 con 0x7fc834166ef0 2024-08-14T06:52:14.769 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.769+0000 7fc83a0bb640 1 -- 172.21.15.98:0/1538418903 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc834166ef0 msgr2=0x7fc8340cde50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.769 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.769+0000 7fc83a0bb640 1 --2- 172.21.15.98:0/1538418903 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc834166ef0 0x7fc8340cde50 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fc82c00ca60 tx=0x7fc82c00cf20 comp rx=0 tx=0).stop 2024-08-14T06:52:14.771 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.771+0000 7fc83a0bb640 1 -- 172.21.15.98:0/1538418903 shutdown_connections 2024-08-14T06:52:14.771 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.771+0000 7fc83a0bb640 1 --2- 172.21.15.98:0/1538418903 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc834166ef0 0x7fc8340cde50 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.771 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.771+0000 7fc83a0bb640 1 --2- 172.21.15.98:0/1538418903 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc8341665a0 0x7fc8340c8e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.772 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.771+0000 7fc83a0bb640 1 --2- 172.21.15.98:0/1538418903 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc83406db60 0x7fc8341fa7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.772 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.771+0000 7fc83a0bb640 1 -- 172.21.15.98:0/1538418903 >> 172.21.15.98:0/1538418903 conn(0x7fc8340d4280 msgr2=0x7fc8340b7f70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.772 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.771+0000 7fc83a0bb640 1 -- 172.21.15.98:0/1538418903 shutdown_connections 2024-08-14T06:52:14.772 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.771+0000 7fc83a0bb640 1 -- 172.21.15.98:0/1538418903 wait complete. 2024-08-14T06:52:14.791 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_verify_scatter', 'True'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.800 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.800+0000 7f3c08ff9640 1 -- 172.21.15.98:0/125781923 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command([{prefix=config set, name=rados_osd_op_timeout}] v 0) -- 0x7f3c00005850 con 0x7f3c1c1fe210 2024-08-14T06:52:14.807 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.806+0000 7f3c0affd640 1 -- 172.21.15.98:0/125781923 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{prefix=config set, name=rados_osd_op_timeout}]=0 v14)=0 v14) ==== 120+0+0 (secure 0 0 0) 0x7f3c1c14f9b0 con 0x7f3c1c1fe210 2024-08-14T06:52:14.809 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.808+0000 7f3c08ff9640 1 -- 172.21.15.98:0/125781923 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3c1c1fe210 msgr2=0x7f3c1c1fd610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.809 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.808+0000 7f3c08ff9640 1 --2- 172.21.15.98:0/125781923 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3c1c1fe210 0x7f3c1c1fd610 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f3c1400b9e0 tx=0x7f3c1400bea0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.809 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.808+0000 7f3c08ff9640 1 -- 172.21.15.98:0/125781923 shutdown_connections 2024-08-14T06:52:14.809 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.808+0000 7f3c08ff9640 1 --2- 172.21.15.98:0/125781923 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3c1c1fe210 0x7f3c1c1fd610 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.809 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.808+0000 7f3c08ff9640 1 --2- 172.21.15.98:0/125781923 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f3c1c15c1b0 0x7f3c1c14f470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.809 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.808+0000 7f3c08ff9640 1 --2- 172.21.15.98:0/125781923 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3c1c072ca0 0x7f3c1c06e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.809 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.808+0000 7f3c08ff9640 1 -- 172.21.15.98:0/125781923 >> 172.21.15.98:0/125781923 conn(0x7f3c1c0cfee0 msgr2=0x7f3c1c162090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.809 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.808+0000 7f3c08ff9640 1 -- 172.21.15.98:0/125781923 shutdown_connections 2024-08-14T06:52:14.810 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.808+0000 7f3c08ff9640 1 -- 172.21.15.98:0/125781923 wait complete. 2024-08-14T06:52:14.816 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.815+0000 7f08ff7fe640 1 -- 172.21.15.98:0/1900960511 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command([{prefix=config set, name=osd_op_complaint_time}] v 0) -- 0x7f08f4005580 con 0x7f090c0e7400 2024-08-14T06:52:14.817 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.816+0000 7f9013fff640 1 -- 172.21.15.98:0/3206283197 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command([{prefix=config set, name=osd_op_complaint_time}] v 0) -- 0x7f90280defe0 con 0x7f90280df7f0 2024-08-14T06:52:14.822 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.822+0000 7f0909ffb640 1 -- 172.21.15.98:0/1900960511 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{prefix=config set, name=osd_op_complaint_time}]=0 v15)=0 v15) ==== 121+0+0 (secure 0 0 0) 0x7f09140bf440 con 0x7f090c0e7400 2024-08-14T06:52:14.825 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.823+0000 7f08ff7fe640 1 -- 172.21.15.98:0/1900960511 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f090c0e7400 msgr2=0x7f090c0dfa60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.825 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.823+0000 7f08ff7fe640 1 --2- 172.21.15.98:0/1900960511 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f090c0e7400 0x7f090c0dfa60 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f09140519c0 tx=0x7f09140b8d80 comp rx=0 tx=0).stop 2024-08-14T06:52:14.825 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.823+0000 7f08ff7fe640 1 -- 172.21.15.98:0/1900960511 shutdown_connections 2024-08-14T06:52:14.825 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.823+0000 7f08ff7fe640 1 --2- 172.21.15.98:0/1900960511 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f090c175c90 0x7f090c0e2480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.825 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.823+0000 7f08ff7fe640 1 --2- 172.21.15.98:0/1900960511 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f090c0e7400 0x7f090c0dfa60 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.825 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.823+0000 7f08ff7fe640 1 --2- 172.21.15.98:0/1900960511 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f090c052350 0x7f090c0d7770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.826 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.823+0000 7f08ff7fe640 1 -- 172.21.15.98:0/1900960511 >> 172.21.15.98:0/1900960511 conn(0x7f090c01d990 msgr2=0x7f090c0e1cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.826 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.823+0000 7f08ff7fe640 1 -- 172.21.15.98:0/1900960511 shutdown_connections 2024-08-14T06:52:14.826 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.823+0000 7f08ff7fe640 1 -- 172.21.15.98:0/1900960511 wait complete. 2024-08-14T06:52:14.828 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.827+0000 7f90267fc640 1 -- 172.21.15.98:0/3206283197 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{prefix=config set, name=osd_op_complaint_time}]=0 v16)=0 v16) ==== 121+0+0 (secure 0 0 0) 0x7f9030074790 con 0x7f90280df7f0 2024-08-14T06:52:14.830 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.829+0000 7f9013fff640 1 -- 172.21.15.98:0/3206283197 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f90280df7f0 msgr2=0x7f902804fde0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.831 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.829+0000 7f9013fff640 1 --2- 172.21.15.98:0/3206283197 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f90280df7f0 0x7f902804fde0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f9030052510 tx=0x7f903006acb0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.831 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.829+0000 7f9013fff640 1 -- 172.21.15.98:0/3206283197 shutdown_connections 2024-08-14T06:52:14.831 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.829+0000 7f9013fff640 1 --2- 172.21.15.98:0/3206283197 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f90280df7f0 0x7f902804fde0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.831 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.829+0000 7f9013fff640 1 --2- 172.21.15.98:0/3206283197 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9028051990 0x7f902801ecc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.831 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.829+0000 7f9013fff640 1 --2- 172.21.15.98:0/3206283197 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9028006810 0x7f90280da620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.831 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.829+0000 7f9013fff640 1 -- 172.21.15.98:0/3206283197 >> 172.21.15.98:0/3206283197 conn(0x7f902801e250 msgr2=0x7f9028005920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.831 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.830+0000 7f9013fff640 1 -- 172.21.15.98:0/3206283197 shutdown_connections 2024-08-14T06:52:14.831 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.830+0000 7f9013fff640 1 -- 172.21.15.98:0/3206283197 wait complete. 2024-08-14T06:52:14.839 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.839+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_command([{prefix=config set, name=rados_mon_op_timeout}] v 0) -- 0x7f593c0b5790 con 0x7f593c15aab0 2024-08-14T06:52:14.845 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.844+0000 7f5931ffb640 1 -- 172.21.15.98:0/3713758045 <== mon.1 v2:172.21.15.191:3300/0 7 ==== mon_command_ack([{prefix=config set, name=rados_mon_op_timeout}]=0 v17)=0 v17) ==== 120+0+0 (secure 0 0 0) 0x7f592c00fcf0 con 0x7f593c15aab0 2024-08-14T06:52:14.847 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.846+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f593c15aab0 msgr2=0x7f593c1431b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.847 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.846+0000 7f59424fb640 1 --2- 172.21.15.98:0/3713758045 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f593c15aab0 0x7f593c1431b0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f593c06ff30 tx=0x7f592c00cd50 comp rx=0 tx=0).stop 2024-08-14T06:52:14.848 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.846+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 shutdown_connections 2024-08-14T06:52:14.848 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.846+0000 7f59424fb640 1 --2- 172.21.15.98:0/3713758045 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f593c15b400 0x7f593c145a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.848 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.846+0000 7f59424fb640 1 --2- 172.21.15.98:0/3713758045 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f593c15aab0 0x7f593c1431b0 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.848 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.846+0000 7f59424fb640 1 --2- 172.21.15.98:0/3713758045 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f593c0cad00 0x7f593c1f9400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.848 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.846+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 >> 172.21.15.98:0/3713758045 conn(0x7f593c0b1e60 msgr2=0x7f593c158660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.848 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.846+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 shutdown_connections 2024-08-14T06:52:14.848 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.846+0000 7f59424fb640 1 -- 172.21.15.98:0/3713758045 wait complete. 2024-08-14T06:52:14.856 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.856+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command([{prefix=config set, name=mon_warn_on_pool_no_app}] v 0) -- 0x7f450c09df80 con 0x7f450c06fdf0 2024-08-14T06:52:14.860 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.859+0000 7f0575eb6640 1 -- 172.21.15.98:0/1381416276 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command([{prefix=config set, name=mds_verify_scatter}] v 0) -- 0x7f0550005580 con 0x7f05700aec30 2024-08-14T06:52:14.861 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.860+0000 7f45097fa640 1 -- 172.21.15.98:0/4212822085 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_warn_on_pool_no_app}]=0 v18)=0 v18) ==== 125+0+0 (secure 0 0 0) 0x7f450001aa70 con 0x7f450c06fdf0 2024-08-14T06:52:14.863 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.861+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f450c06fdf0 msgr2=0x7f450c0b2d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.863 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.861+0000 7f4512d38640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f450c06fdf0 0x7f450c0b2d10 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f450000b420 tx=0x7f450000b8e0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.863 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.861+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 shutdown_connections 2024-08-14T06:52:14.863 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.861+0000 7f4512d38640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f450c073f80 0x7f450c153cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.863 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.861+0000 7f4512d38640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f450c072e80 0x7f450c152e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.863 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.861+0000 7f4512d38640 1 --2- 172.21.15.98:0/4212822085 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f450c06fdf0 0x7f450c0b2d10 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.863 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.862+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 >> 172.21.15.98:0/4212822085 conn(0x7f450c0cfee0 msgr2=0x7f450c0d02f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.863 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.862+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 shutdown_connections 2024-08-14T06:52:14.863 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.862+0000 7f4512d38640 1 -- 172.21.15.98:0/4212822085 wait complete. 2024-08-14T06:52:14.866 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.865+0000 7f056d7fa640 1 -- 172.21.15.98:0/1381416276 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_verify_scatter}]=0 v19)=0 v19) ==== 119+0+0 (secure 0 0 0) 0x7f0568012940 con 0x7f05700aec30 2024-08-14T06:52:14.868 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.867+0000 7f0575eb6640 1 -- 172.21.15.98:0/1381416276 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f05700aec30 msgr2=0x7f05700cc8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:14.868 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.867+0000 7f0575eb6640 1 --2- 172.21.15.98:0/1381416276 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f05700aec30 0x7f05700cc8a0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f056800c730 tx=0x7f056800cbf0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.868 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.867+0000 7f0575eb6640 1 -- 172.21.15.98:0/1381416276 shutdown_connections 2024-08-14T06:52:14.868 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.867+0000 7f0575eb6640 1 --2- 172.21.15.98:0/1381416276 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f0570162410 0x7f05701fce40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.868 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.867+0000 7f0575eb6640 1 --2- 172.21.15.98:0/1381416276 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f05700cda80 0x7f05701fc900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.868 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.867+0000 7f0575eb6640 1 --2- 172.21.15.98:0/1381416276 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f05700aec30 0x7f05700cc8a0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:14.868 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.867+0000 7f0575eb6640 1 -- 172.21.15.98:0/1381416276 >> 172.21.15.98:0/1381416276 conn(0x7f05700b1e60 msgr2=0x7f0570153ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:14.869 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.867+0000 7f0575eb6640 1 -- 172.21.15.98:0/1381416276 shutdown_connections 2024-08-14T06:52:14.869 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:14.867+0000 7f0575eb6640 1 -- 172.21.15.98:0/1381416276 wait complete. 2024-08-14T06:52:14.879 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'osd_op_complaint_time', '180'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.879 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'rados_mon_op_timeout', '900'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.880 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'rados_osd_op_timeout', '900'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.880 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mon', 'mon_op_complaint_time', '120'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.880 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mon', 'mon_warn_on_pool_no_app', 'False'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.880 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'osd', 'osd_op_complaint_time', '180'], check_status=True, hostname='smithi098') 2024-08-14T06:52:14.880 INFO:tasks.ceph:Setting crush tunables to default 2024-08-14T06:52:14.880 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph osd crush tunables default 2024-08-14T06:52:15.036 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.035+0000 7f79c7f1f640 1 -- 172.21.15.98:0/460889356 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f79c00d0530 msgr2=0x7f79c015ffb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.036 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.035+0000 7f79c7f1f640 1 --2- 172.21.15.98:0/460889356 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f79c00d0530 0x7f79c015ffb0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f79b0009830 tx=0x7f79b002a480 comp rx=0 tx=0).stop 2024-08-14T06:52:15.036 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.035+0000 7f79c7f1f640 1 -- 172.21.15.98:0/460889356 shutdown_connections 2024-08-14T06:52:15.036 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.035+0000 7f79c7f1f640 1 --2- 172.21.15.98:0/460889356 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f79c00d0530 0x7f79c015ffb0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.036 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.035+0000 7f79c7f1f640 1 --2- 172.21.15.98:0/460889356 >> v2:172.21.15.191:3300/0 conn(0x7f79c00d0940 0x7f79c01604f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.036 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.035+0000 7f79c7f1f640 1 -- 172.21.15.98:0/460889356 >> 172.21.15.98:0/460889356 conn(0x7f79c00b2b30 msgr2=0x7f79c00b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:15.036 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.035+0000 7f79c7f1f640 1 -- 172.21.15.98:0/460889356 shutdown_connections 2024-08-14T06:52:15.036 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.035+0000 7f79c7f1f640 1 -- 172.21.15.98:0/460889356 wait complete. 2024-08-14T06:52:15.037 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.036+0000 7f79c7f1f640 1 Processor -- start 2024-08-14T06:52:15.037 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.037+0000 7f79c7f1f640 1 -- start start 2024-08-14T06:52:15.040 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c7f1f640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f79c00d0530 0x7f79c00b5310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:15.040 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c7f1f640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f79c00d0940 0x7f79c01f6bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:15.040 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c7f1f640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f79c00c57e0 0x7f79c00b1930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:15.040 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c7f1f640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f79c0073f90 con 0x7f79c00d0530 2024-08-14T06:52:15.040 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c7f1f640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f79c0073e10 con 0x7f79c00d0940 2024-08-14T06:52:15.040 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c7f1f640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f79c0074110 con 0x7f79c00c57e0 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c5cbb640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f79c00d0530 0x7f79c00b5310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c5cbb640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f79c00d0530 0x7f79c00b5310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:38714/0 (socket says 172.21.15.98:38714) 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c5cbb640 1 -- 172.21.15.98:0/611941781 learned_addr learned my addr 172.21.15.98:0/611941781 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c54ba640 1 --2- 172.21.15.98:0/611941781 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f79c00d0940 0x7f79c01f6bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c64bc640 1 --2- 172.21.15.98:0/611941781 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f79c00c57e0 0x7f79c00b1930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c5cbb640 1 -- 172.21.15.98:0/611941781 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f79c00c57e0 msgr2=0x7f79c00b1930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c5cbb640 1 --2- 172.21.15.98:0/611941781 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f79c00c57e0 0x7f79c00b1930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c5cbb640 1 -- 172.21.15.98:0/611941781 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f79c00d0940 msgr2=0x7f79c01f6bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c5cbb640 1 --2- 172.21.15.98:0/611941781 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f79c00d0940 0x7f79c01f6bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c5cbb640 1 -- 172.21.15.98:0/611941781 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f79c01f7100 con 0x7f79c00d0530 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.038+0000 7f79c5cbb640 1 --2- 172.21.15.98:0/611941781 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f79c00d0530 0x7f79c00b5310 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f79bc00c400 tx=0x7f79bc00c800 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:15.041 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.039+0000 7f79b67fc640 1 -- 172.21.15.98:0/611941781 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f79bc011790 con 0x7f79c00d0530 2024-08-14T06:52:15.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.039+0000 7f79b67fc640 1 -- 172.21.15.98:0/611941781 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f79bc017070 con 0x7f79c00d0530 2024-08-14T06:52:15.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.039+0000 7f79b67fc640 1 -- 172.21.15.98:0/611941781 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f79bc011cc0 con 0x7f79c00d0530 2024-08-14T06:52:15.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.039+0000 7f79c7f1f640 1 -- 172.21.15.98:0/611941781 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f79c00c64c0 con 0x7f79c00d0530 2024-08-14T06:52:15.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.039+0000 7f79c7f1f640 1 -- 172.21.15.98:0/611941781 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f79c00c4fe0 con 0x7f79c00d0530 2024-08-14T06:52:15.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.039+0000 7f79b67fc640 1 -- 172.21.15.98:0/611941781 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f79bc005020 con 0x7f79c00d0530 2024-08-14T06:52:15.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.039+0000 7f79b67fc640 1 -- 172.21.15.98:0/611941781 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f79bc01c460 con 0x7f79c00d0530 2024-08-14T06:52:15.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.040+0000 7f79c7f1f640 1 -- 172.21.15.98:0/611941781 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f79c0074630 con 0x7f79c00d0530 2024-08-14T06:52:15.043 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.042+0000 7f79b67fc640 1 -- 172.21.15.98:0/611941781 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f79bc021020 con 0x7f79c00d0530 2024-08-14T06:52:15.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.111+0000 7f79c7f1f640 1 -- 172.21.15.98:0/611941781 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) -- 0x7f79c00cfc90 con 0x7f79c00d0530 2024-08-14T06:52:15.114 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.114+0000 7f79b67fc640 1 -- 172.21.15.98:0/611941781 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v2) ==== 124+0+0 (secure 0 0 0) 0x7f79c01f7100 con 0x7f79c00d0530 2024-08-14T06:52:15.115 INFO:teuthology.orchestra.run.smithi098.stderr:adjusted tunables profile to default 2024-08-14T06:52:15.116 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.115+0000 7f79c7f1f640 1 -- 172.21.15.98:0/611941781 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f79c00d0530 msgr2=0x7f79c00b5310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.117 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.115+0000 7f79c7f1f640 1 --2- 172.21.15.98:0/611941781 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f79c00d0530 0x7f79c00b5310 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f79bc00c400 tx=0x7f79bc00c800 comp rx=0 tx=0).stop 2024-08-14T06:52:15.117 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.115+0000 7f79c7f1f640 1 -- 172.21.15.98:0/611941781 shutdown_connections 2024-08-14T06:52:15.117 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.115+0000 7f79c7f1f640 1 --2- 172.21.15.98:0/611941781 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f79c00c57e0 0x7f79c00b1930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.117 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.115+0000 7f79c7f1f640 1 --2- 172.21.15.98:0/611941781 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f79c00d0940 0x7f79c01f6bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.117 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.115+0000 7f79c7f1f640 1 --2- 172.21.15.98:0/611941781 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f79c00d0530 0x7f79c00b5310 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.117 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.115+0000 7f79c7f1f640 1 -- 172.21.15.98:0/611941781 >> 172.21.15.98:0/611941781 conn(0x7f79c00b2b30 msgr2=0x7f79c00b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:15.117 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.116+0000 7f79c7f1f640 1 -- 172.21.15.98:0/611941781 shutdown_connections 2024-08-14T06:52:15.117 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:15.116+0000 7f79c7f1f640 1 -- 172.21.15.98:0/611941781 wait complete. 2024-08-14T06:52:15.131 INFO:tasks.ceph:check_enable_crimson: False 2024-08-14T06:52:15.131 INFO:tasks.ceph:Starting osd daemons in cluster ceph... 2024-08-14T06:52:15.132 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:15.132 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-0/fsid of=/dev/stdout 2024-08-14T06:52:15.157 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:15.157 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-1/fsid of=/dev/stdout 2024-08-14T06:52:15.223 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:15.224 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-2/fsid of=/dev/stdout 2024-08-14T06:52:15.288 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-08-14T06:52:15.288 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-3/fsid of=/dev/stdout 2024-08-14T06:52:15.353 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:15.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-4/fsid of=/dev/stdout 2024-08-14T06:52:15.378 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:15.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-5/fsid of=/dev/stdout 2024-08-14T06:52:15.444 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:15.444 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-6/fsid of=/dev/stdout 2024-08-14T06:52:15.509 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-08-14T06:52:15.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-7/fsid of=/dev/stdout 2024-08-14T06:52:15.575 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new 273f37e4-1d4c-46cd-96bc-3d7fce969572 0 2024-08-14T06:52:15.728 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.727+0000 7f7c48af9640 1 -- 172.21.15.191:0/1442579961 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7c440b6030 msgr2=0x7f7c44160e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.729 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.727+0000 7f7c48af9640 1 --2- 172.21.15.191:0/1442579961 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7c440b6030 0x7f7c44160e30 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f7c3c009ca0 tx=0x7f7c3c02aa90 comp rx=0 tx=0).stop 2024-08-14T06:52:15.729 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.727+0000 7f7c48af9640 1 -- 172.21.15.191:0/1442579961 shutdown_connections 2024-08-14T06:52:15.729 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.727+0000 7f7c48af9640 1 --2- 172.21.15.191:0/1442579961 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7c440b6030 0x7f7c44160e30 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.729 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.727+0000 7f7c48af9640 1 -- 172.21.15.191:0/1442579961 >> 172.21.15.191:0/1442579961 conn(0x7f7c440b3b70 msgr2=0x7f7c440b3f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:15.729 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.727+0000 7f7c48af9640 1 -- 172.21.15.191:0/1442579961 shutdown_connections 2024-08-14T06:52:15.729 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.727+0000 7f7c48af9640 1 -- 172.21.15.191:0/1442579961 wait complete. 2024-08-14T06:52:15.730 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.729+0000 7f7c48af9640 1 Processor -- start 2024-08-14T06:52:15.730 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.729+0000 7f7c48af9640 1 -- start start 2024-08-14T06:52:15.733 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c48af9640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7c440b30d0 0x7f7c44152d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:15.733 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c48af9640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7c440b4690 0x7f7c44153250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:15.734 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c48af9640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7c440b6030 0x7f7c44153790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:15.734 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c48af9640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f7c441679e0 con 0x7f7c440b30d0 2024-08-14T06:52:15.734 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c48af9640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f7c44161580 con 0x7f7c440b6030 2024-08-14T06:52:15.734 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c48af9640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f7c44167b60 con 0x7f7c440b4690 2024-08-14T06:52:15.734 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c42d9d640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7c440b6030 0x7f7c44153790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:15.734 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c41d9b640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7c440b4690 0x7f7c44153250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:15.734 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c41d9b640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7c440b4690 0x7f7c44153250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3301/0 says I am v2:172.21.15.191:37668/0 (socket says 172.21.15.191:37668) 2024-08-14T06:52:15.734 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c42d9d640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7c440b6030 0x7f7c44153790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3300/0 says I am v2:172.21.15.191:58478/0 (socket says 172.21.15.191:58478) 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c41d9b640 1 -- 172.21.15.191:0/1529637191 learned_addr learned my addr 172.21.15.191:0/1529637191 (peer_addr_for_me v2:172.21.15.191:0/0) 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c4259c640 1 --2- 172.21.15.191:0/1529637191 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7c440b30d0 0x7f7c44152d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c41d9b640 1 -- 172.21.15.191:0/1529637191 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7c440b6030 msgr2=0x7f7c44153790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c41d9b640 1 --2- 172.21.15.191:0/1529637191 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7c440b6030 0x7f7c44153790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c41d9b640 1 -- 172.21.15.191:0/1529637191 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7c440b30d0 msgr2=0x7f7c44152d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c41d9b640 1 --2- 172.21.15.191:0/1529637191 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7c440b30d0 0x7f7c44152d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c41d9b640 1 -- 172.21.15.191:0/1529637191 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7c4415dc10 con 0x7f7c440b4690 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c42d9d640 1 --2- 172.21.15.191:0/1529637191 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7c440b6030 0x7f7c44153790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c4259c640 1 --2- 172.21.15.191:0/1529637191 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7c440b30d0 0x7f7c44152d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.731+0000 7f7c41d9b640 1 --2- 172.21.15.191:0/1529637191 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7c440b4690 0x7f7c44153250 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f7c2c001eb0 tx=0x7f7c2c00c890 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.732+0000 7f7c3affd640 1 -- 172.21.15.191:0/1529637191 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c2c017070 con 0x7f7c440b4690 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.732+0000 7f7c3affd640 1 -- 172.21.15.191:0/1529637191 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7c2c0053c0 con 0x7f7c440b4690 2024-08-14T06:52:15.735 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.732+0000 7f7c3affd640 1 -- 172.21.15.191:0/1529637191 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c2c003830 con 0x7f7c440b4690 2024-08-14T06:52:15.736 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.732+0000 7f7c48af9640 1 -- 172.21.15.191:0/1529637191 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7c4415def0 con 0x7f7c440b4690 2024-08-14T06:52:15.736 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.732+0000 7f7c48af9640 1 -- 172.21.15.191:0/1529637191 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f7c441571d0 con 0x7f7c440b4690 2024-08-14T06:52:15.736 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.732+0000 7f7c3affd640 1 -- 172.21.15.191:0/1529637191 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f7c2c01a070 con 0x7f7c440b4690 2024-08-14T06:52:15.736 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.732+0000 7f7c3affd640 1 -- 172.21.15.191:0/1529637191 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(2..2 src has 1..2) ==== 725+0+0 (secure 0 0 0) 0x7f7c2c01c780 con 0x7f7c440b4690 2024-08-14T06:52:15.736 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.733+0000 7f7c48af9640 1 -- 172.21.15.191:0/1529637191 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7c4409ad00 con 0x7f7c440b4690 2024-08-14T06:52:15.736 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.735+0000 7f7c3affd640 1 -- 172.21.15.191:0/1529637191 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f7c2c021080 con 0x7f7c440b4690 2024-08-14T06:52:15.808 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.807+0000 7f7c48af9640 1 -- 172.21.15.191:0/1529637191 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "osd new", "uuid": "273f37e4-1d4c-46cd-96bc-3d7fce969572", "id": 0} v 0) -- 0x7f7c440cc6d0 con 0x7f7c440b4690 2024-08-14T06:52:15.811 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.811+0000 7f7c3affd640 1 -- 172.21.15.191:0/1529637191 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "273f37e4-1d4c-46cd-96bc-3d7fce969572", "id": 0}]=0 v3) ==== 112+0+2 (secure 0 0 0) 0x7f7c2c011dc0 con 0x7f7c440b4690 2024-08-14T06:52:15.812 INFO:teuthology.orchestra.run.smithi191.stdout:0 2024-08-14T06:52:15.813 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.812+0000 7f7c48af9640 1 -- 172.21.15.191:0/1529637191 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7c440b4690 msgr2=0x7f7c44153250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.813 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.812+0000 7f7c48af9640 1 --2- 172.21.15.191:0/1529637191 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7c440b4690 0x7f7c44153250 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f7c2c001eb0 tx=0x7f7c2c00c890 comp rx=0 tx=0).stop 2024-08-14T06:52:15.813 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.812+0000 7f7c48af9640 1 -- 172.21.15.191:0/1529637191 shutdown_connections 2024-08-14T06:52:15.813 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.812+0000 7f7c48af9640 1 --2- 172.21.15.191:0/1529637191 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f7c440b6030 0x7f7c44153790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.813 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.812+0000 7f7c48af9640 1 --2- 172.21.15.191:0/1529637191 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f7c440b4690 0x7f7c44153250 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.813 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.812+0000 7f7c48af9640 1 --2- 172.21.15.191:0/1529637191 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f7c440b30d0 0x7f7c44152d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.813 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.812+0000 7f7c48af9640 1 -- 172.21.15.191:0/1529637191 >> 172.21.15.191:0/1529637191 conn(0x7f7c440b3b70 msgr2=0x7f7c4415bf50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:15.813 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.812+0000 7f7c48af9640 1 -- 172.21.15.191:0/1529637191 shutdown_connections 2024-08-14T06:52:15.813 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.812+0000 7f7c48af9640 1 -- 172.21.15.191:0/1529637191 wait complete. 2024-08-14T06:52:15.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new ebd9bc08-f436-4e26-a0ba-d79a2220ba5e 1 2024-08-14T06:52:15.939 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.937+0000 7f518cc84640 1 -- 172.21.15.191:0/873081844 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5188073ea0 msgr2=0x7f5188151b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.939 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.937+0000 7f518cc84640 1 --2- 172.21.15.191:0/873081844 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5188073ea0 0x7f5188151b40 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f517c005fd0 tx=0x7f517c02a560 comp rx=0 tx=0).stop 2024-08-14T06:52:15.939 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.938+0000 7f518cc84640 1 -- 172.21.15.191:0/873081844 shutdown_connections 2024-08-14T06:52:15.939 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.938+0000 7f518cc84640 1 --2- 172.21.15.191:0/873081844 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5188073ea0 0x7f5188151b40 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.939 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.938+0000 7f518cc84640 1 -- 172.21.15.191:0/873081844 >> 172.21.15.191:0/873081844 conn(0x7f51880b2b30 msgr2=0x7f51880b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:15.939 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.938+0000 7f518cc84640 1 -- 172.21.15.191:0/873081844 shutdown_connections 2024-08-14T06:52:15.939 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.938+0000 7f518cc84640 1 -- 172.21.15.191:0/873081844 wait complete. 2024-08-14T06:52:15.940 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.940+0000 7f518cc84640 1 Processor -- start 2024-08-14T06:52:15.940 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.940+0000 7f518cc84640 1 -- start start 2024-08-14T06:52:15.943 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f518cc84640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f51880cc190 0x7f518814fd00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:15.943 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f518cc84640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5188152080 0x7f51881f54b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f518cc84640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5188150530 0x7f51881f59f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f518cc84640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f518815c210 con 0x7f5188150530 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f518cc84640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f518815c390 con 0x7f5188152080 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f518cc84640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f518815c510 con 0x7f51880cc190 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f5185d9b640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5188152080 0x7f51881f54b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f518659c640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f51880cc190 0x7f518814fd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f5185d9b640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5188152080 0x7f51881f54b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3300/0 says I am v2:172.21.15.191:58510/0 (socket says 172.21.15.191:58510) 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f5185d9b640 1 -- 172.21.15.191:0/462534810 learned_addr learned my addr 172.21.15.191:0/462534810 (peer_addr_for_me v2:172.21.15.191:0/0) 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f5186d9d640 1 --2- 172.21.15.191:0/462534810 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5188150530 0x7f51881f59f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f5185d9b640 1 -- 172.21.15.191:0/462534810 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f51880cc190 msgr2=0x7f518814fd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.944 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f5185d9b640 1 --2- 172.21.15.191:0/462534810 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f51880cc190 0x7f518814fd00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f5185d9b640 1 -- 172.21.15.191:0/462534810 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5188150530 msgr2=0x7f51881f59f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f5185d9b640 1 --2- 172.21.15.191:0/462534810 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5188150530 0x7f51881f59f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f5185d9b640 1 -- 172.21.15.191:0/462534810 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f51881f2910 con 0x7f5188152080 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.941+0000 7f518659c640 1 --2- 172.21.15.191:0/462534810 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f51880cc190 0x7f518814fd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.942+0000 7f5185d9b640 1 --2- 172.21.15.191:0/462534810 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5188152080 0x7f51881f54b0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f517c006010 tx=0x7f517c0039e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.942+0000 7f5176ffd640 1 -- 172.21.15.191:0/462534810 <== mon.1 v2:172.21.15.191:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f517c03e030 con 0x7f5188152080 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.942+0000 7f5176ffd640 1 -- 172.21.15.191:0/462534810 <== mon.1 v2:172.21.15.191:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f517c0047a0 con 0x7f5188152080 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.942+0000 7f518cc84640 1 -- 172.21.15.191:0/462534810 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f51881f2e70 con 0x7f5188152080 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.942+0000 7f5176ffd640 1 -- 172.21.15.191:0/462534810 <== mon.1 v2:172.21.15.191:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f517c035040 con 0x7f5188152080 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.942+0000 7f518cc84640 1 -- 172.21.15.191:0/462534810 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f51880c57e0 con 0x7f5188152080 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.942+0000 7f5176ffd640 1 -- 172.21.15.191:0/462534810 <== mon.1 v2:172.21.15.191:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f517c041030 con 0x7f5188152080 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.942+0000 7f5176ffd640 1 -- 172.21.15.191:0/462534810 <== mon.1 v2:172.21.15.191:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 840+0+0 (secure 0 0 0) 0x7f517c031400 con 0x7f5188152080 2024-08-14T06:52:15.945 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.943+0000 7f518cc84640 1 -- 172.21.15.191:0/462534810 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5188151b40 con 0x7f5188152080 2024-08-14T06:52:15.946 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:15.945+0000 7f5176ffd640 1 -- 172.21.15.191:0/462534810 <== mon.1 v2:172.21.15.191:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f51881f2910 con 0x7f5188152080 2024-08-14T06:52:16.019 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.018+0000 7f518cc84640 1 -- 172.21.15.191:0/462534810 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_command({"prefix": "osd new", "uuid": "ebd9bc08-f436-4e26-a0ba-d79a2220ba5e", "id": 1} v 0) -- 0x7f5188074aa0 con 0x7f5188152080 2024-08-14T06:52:16.022 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.021+0000 7f5176ffd640 1 -- 172.21.15.191:0/462534810 <== mon.1 v2:172.21.15.191:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "ebd9bc08-f436-4e26-a0ba-d79a2220ba5e", "id": 1}]=0 v4) ==== 112+0+2 (secure 0 0 0) 0x7f517c043410 con 0x7f5188152080 2024-08-14T06:52:16.022 INFO:teuthology.orchestra.run.smithi191.stdout:1 2024-08-14T06:52:16.024 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.022+0000 7f518cc84640 1 -- 172.21.15.191:0/462534810 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5188152080 msgr2=0x7f51881f54b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.024 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.022+0000 7f518cc84640 1 --2- 172.21.15.191:0/462534810 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5188152080 0x7f51881f54b0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f517c006010 tx=0x7f517c0039e0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.024 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.022+0000 7f518cc84640 1 -- 172.21.15.191:0/462534810 shutdown_connections 2024-08-14T06:52:16.024 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.022+0000 7f518cc84640 1 --2- 172.21.15.191:0/462534810 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5188150530 0x7f51881f59f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.024 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.022+0000 7f518cc84640 1 --2- 172.21.15.191:0/462534810 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5188152080 0x7f51881f54b0 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.024 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.022+0000 7f518cc84640 1 --2- 172.21.15.191:0/462534810 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f51880cc190 0x7f518814fd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.024 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.022+0000 7f518cc84640 1 -- 172.21.15.191:0/462534810 >> 172.21.15.191:0/462534810 conn(0x7f51880b2b30 msgr2=0x7f51880b2320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:16.024 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.023+0000 7f518cc84640 1 -- 172.21.15.191:0/462534810 shutdown_connections 2024-08-14T06:52:16.024 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.023+0000 7f518cc84640 1 -- 172.21.15.191:0/462534810 wait complete. 2024-08-14T06:52:16.034 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new 9979a52a-0573-4019-a471-2ad384d88176 2 2024-08-14T06:52:16.149 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.148+0000 7facadf84640 1 -- 172.21.15.191:0/748838819 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faca80cc190 msgr2=0x7faca8073790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.150 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.148+0000 7facadf84640 1 --2- 172.21.15.191:0/748838819 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faca80cc190 0x7faca8073790 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fac9c009830 tx=0x7fac9c02a4a0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.150 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.148+0000 7facadf84640 1 -- 172.21.15.191:0/748838819 shutdown_connections 2024-08-14T06:52:16.150 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.148+0000 7facadf84640 1 --2- 172.21.15.191:0/748838819 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faca80cc190 0x7faca8073790 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.150 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.148+0000 7facadf84640 1 -- 172.21.15.191:0/748838819 >> 172.21.15.191:0/748838819 conn(0x7faca80b2b30 msgr2=0x7faca80b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:16.150 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.148+0000 7facadf84640 1 -- 172.21.15.191:0/748838819 shutdown_connections 2024-08-14T06:52:16.150 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.148+0000 7facadf84640 1 -- 172.21.15.191:0/748838819 wait complete. 2024-08-14T06:52:16.150 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.150+0000 7facadf84640 1 Processor -- start 2024-08-14T06:52:16.151 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.150+0000 7facadf84640 1 -- start start 2024-08-14T06:52:16.154 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7facadf84640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faca8073cd0 0x7faca80b6640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.154 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7facadf84640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faca80cc190 0x7faca81510d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.154 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7facadf84640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faca8152250 0x7faca8154170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.154 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7facadf84640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7faca815c210 con 0x7faca8073cd0 2024-08-14T06:52:16.154 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7facadf84640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7faca80d00e0 con 0x7faca80cc190 2024-08-14T06:52:16.154 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7facadf84640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7faca815c390 con 0x7faca8152250 2024-08-14T06:52:16.154 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7faca6ffd640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faca80cc190 0x7faca81510d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.154 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7faca7fff640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faca8152250 0x7faca8154170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.154 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7faca6ffd640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faca80cc190 0x7faca81510d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3300/0 says I am v2:172.21.15.191:58524/0 (socket says 172.21.15.191:58524) 2024-08-14T06:52:16.154 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7faca7fff640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faca8152250 0x7faca8154170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3301/0 says I am v2:172.21.15.191:37684/0 (socket says 172.21.15.191:37684) 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7faca6ffd640 1 -- 172.21.15.191:0/1173792984 learned_addr learned my addr 172.21.15.191:0/1173792984 (peer_addr_for_me v2:172.21.15.191:0/0) 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.151+0000 7faca77fe640 1 --2- 172.21.15.191:0/1173792984 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faca8073cd0 0x7faca80b6640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7faca7fff640 1 -- 172.21.15.191:0/1173792984 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faca80cc190 msgr2=0x7faca81510d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7faca7fff640 1 --2- 172.21.15.191:0/1173792984 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faca80cc190 0x7faca81510d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7faca7fff640 1 -- 172.21.15.191:0/1173792984 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faca8073cd0 msgr2=0x7faca80b6640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7faca7fff640 1 --2- 172.21.15.191:0/1173792984 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faca8073cd0 0x7faca80b6640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7faca7fff640 1 -- 172.21.15.191:0/1173792984 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faca80b6d70 con 0x7faca8152250 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7faca6ffd640 1 --2- 172.21.15.191:0/1173792984 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faca80cc190 0x7faca81510d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7faca77fe640 1 --2- 172.21.15.191:0/1173792984 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faca8073cd0 0x7faca80b6640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7faca7fff640 1 --2- 172.21.15.191:0/1173792984 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faca8152250 0x7faca8154170 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7faca000caa0 tx=0x7faca000cf60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7fac8ffff640 1 -- 172.21.15.191:0/1173792984 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faca0016020 con 0x7faca8152250 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7fac8ffff640 1 -- 172.21.15.191:0/1173792984 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7faca000acb0 con 0x7faca8152250 2024-08-14T06:52:16.155 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7facadf84640 1 -- 172.21.15.191:0/1173792984 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faca8153480 con 0x7faca8152250 2024-08-14T06:52:16.156 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7fac8ffff640 1 -- 172.21.15.191:0/1173792984 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faca0012680 con 0x7faca8152250 2024-08-14T06:52:16.156 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7facadf84640 1 -- 172.21.15.191:0/1173792984 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7faca8153a40 con 0x7faca8152250 2024-08-14T06:52:16.156 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7fac8ffff640 1 -- 172.21.15.191:0/1173792984 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7faca0019020 con 0x7faca8152250 2024-08-14T06:52:16.156 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.152+0000 7fac8ffff640 1 -- 172.21.15.191:0/1173792984 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(4..4 src has 1..4) ==== 955+0+0 (secure 0 0 0) 0x7faca001bc50 con 0x7faca8152250 2024-08-14T06:52:16.156 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.153+0000 7facadf84640 1 -- 172.21.15.191:0/1173792984 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faca8071550 con 0x7faca8152250 2024-08-14T06:52:16.156 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.155+0000 7fac8ffff640 1 -- 172.21.15.191:0/1173792984 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7faca8071550 con 0x7faca8152250 2024-08-14T06:52:16.231 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.230+0000 7facadf84640 1 -- 172.21.15.191:0/1173792984 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "osd new", "uuid": "9979a52a-0573-4019-a471-2ad384d88176", "id": 2} v 0) -- 0x7faca8073790 con 0x7faca8152250 2024-08-14T06:52:16.234 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.234+0000 7fac8ffff640 1 -- 172.21.15.191:0/1173792984 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "9979a52a-0573-4019-a471-2ad384d88176", "id": 2}]=0 v5) ==== 112+0+2 (secure 0 0 0) 0x7faca0020020 con 0x7faca8152250 2024-08-14T06:52:16.235 INFO:teuthology.orchestra.run.smithi191.stdout:2 2024-08-14T06:52:16.237 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.235+0000 7facadf84640 1 -- 172.21.15.191:0/1173792984 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faca8152250 msgr2=0x7faca8154170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.237 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.235+0000 7facadf84640 1 --2- 172.21.15.191:0/1173792984 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faca8152250 0x7faca8154170 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7faca000caa0 tx=0x7faca000cf60 comp rx=0 tx=0).stop 2024-08-14T06:52:16.237 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.235+0000 7facadf84640 1 -- 172.21.15.191:0/1173792984 shutdown_connections 2024-08-14T06:52:16.237 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.235+0000 7facadf84640 1 --2- 172.21.15.191:0/1173792984 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faca8152250 0x7faca8154170 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.237 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.235+0000 7facadf84640 1 --2- 172.21.15.191:0/1173792984 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faca80cc190 0x7faca81510d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.237 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.235+0000 7facadf84640 1 --2- 172.21.15.191:0/1173792984 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faca8073cd0 0x7faca80b6640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.237 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.235+0000 7facadf84640 1 -- 172.21.15.191:0/1173792984 >> 172.21.15.191:0/1173792984 conn(0x7faca80b2b30 msgr2=0x7faca806ca90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:16.237 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.235+0000 7facadf84640 1 -- 172.21.15.191:0/1173792984 shutdown_connections 2024-08-14T06:52:16.237 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.235+0000 7facadf84640 1 -- 172.21.15.191:0/1173792984 wait complete. 2024-08-14T06:52:16.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new aa26d769-fd1e-4963-af87-a7006f935553 3 2024-08-14T06:52:16.362 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.360+0000 7fb58f214640 1 -- 172.21.15.191:0/3827025502 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb588073ea0 msgr2=0x7fb588151b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.362 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.360+0000 7fb58f214640 1 --2- 172.21.15.191:0/3827025502 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb588073ea0 0x7fb588151b40 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fb574005fd0 tx=0x7fb57402a560 comp rx=0 tx=0).stop 2024-08-14T06:52:16.362 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.361+0000 7fb58f214640 1 -- 172.21.15.191:0/3827025502 shutdown_connections 2024-08-14T06:52:16.362 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.361+0000 7fb58f214640 1 --2- 172.21.15.191:0/3827025502 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb588073ea0 0x7fb588151b40 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.363 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.361+0000 7fb58f214640 1 -- 172.21.15.191:0/3827025502 >> 172.21.15.191:0/3827025502 conn(0x7fb5880b2b30 msgr2=0x7fb5880b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:16.363 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.361+0000 7fb58f214640 1 -- 172.21.15.191:0/3827025502 shutdown_connections 2024-08-14T06:52:16.363 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.361+0000 7fb58f214640 1 -- 172.21.15.191:0/3827025502 wait complete. 2024-08-14T06:52:16.363 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.362+0000 7fb58f214640 1 Processor -- start 2024-08-14T06:52:16.363 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.362+0000 7fb58f214640 1 -- start start 2024-08-14T06:52:16.366 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58f214640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb5880cc190 0x7fb588150250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.366 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58f214640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb588152080 0x7fb5880b5380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.366 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58f214640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb5880b58c0 0x7fb5881f5720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.366 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58f214640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7fb58815c210 con 0x7fb5880b58c0 2024-08-14T06:52:16.366 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58f214640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7fb58815c390 con 0x7fb588152080 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58f214640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7fb58815c510 con 0x7fb5880cc190 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb587fff640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb588152080 0x7fb5880b5380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58cfb0640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb5880cc190 0x7fb588150250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb587fff640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb588152080 0x7fb5880b5380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3300/0 says I am v2:172.21.15.191:58542/0 (socket says 172.21.15.191:58542) 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58cfb0640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb5880cc190 0x7fb588150250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3301/0 says I am v2:172.21.15.191:37696/0 (socket says 172.21.15.191:37696) 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb587fff640 1 -- 172.21.15.191:0/2096176261 learned_addr learned my addr 172.21.15.191:0/2096176261 (peer_addr_for_me v2:172.21.15.191:0/0) 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58d7b1640 1 --2- 172.21.15.191:0/2096176261 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb5880b58c0 0x7fb5881f5720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58cfb0640 1 -- 172.21.15.191:0/2096176261 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb588152080 msgr2=0x7fb5880b5380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58cfb0640 1 --2- 172.21.15.191:0/2096176261 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb588152080 0x7fb5880b5380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58cfb0640 1 -- 172.21.15.191:0/2096176261 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb5880b58c0 msgr2=0x7fb5881f5720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58cfb0640 1 --2- 172.21.15.191:0/2096176261 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb5880b58c0 0x7fb5881f5720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58cfb0640 1 -- 172.21.15.191:0/2096176261 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb588150790 con 0x7fb5880cc190 2024-08-14T06:52:16.367 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb587fff640 1 --2- 172.21.15.191:0/2096176261 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb588152080 0x7fb5880b5380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:16.368 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.364+0000 7fb58cfb0640 1 --2- 172.21.15.191:0/2096176261 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb5880cc190 0x7fb588150250 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fb57c00c3f0 tx=0x7fb57c00c8b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:16.368 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.365+0000 7fb5857fa640 1 -- 172.21.15.191:0/2096176261 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb57c017070 con 0x7fb5880cc190 2024-08-14T06:52:16.368 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.365+0000 7fb5857fa640 1 -- 172.21.15.191:0/2096176261 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb57c0053c0 con 0x7fb5880cc190 2024-08-14T06:52:16.368 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.365+0000 7fb58f214640 1 -- 172.21.15.191:0/2096176261 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb5880b6b60 con 0x7fb5880cc190 2024-08-14T06:52:16.368 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.365+0000 7fb5857fa640 1 -- 172.21.15.191:0/2096176261 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb57c003850 con 0x7fb5880cc190 2024-08-14T06:52:16.368 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.365+0000 7fb58f214640 1 -- 172.21.15.191:0/2096176261 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fb5880b4b80 con 0x7fb5880cc190 2024-08-14T06:52:16.368 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.365+0000 7fb5857fa640 1 -- 172.21.15.191:0/2096176261 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fb57c01a070 con 0x7fb5880cc190 2024-08-14T06:52:16.368 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.365+0000 7fb5857fa640 1 -- 172.21.15.191:0/2096176261 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(5..5 src has 1..5) ==== 1070+0+0 (secure 0 0 0) 0x7fb57c011c10 con 0x7fb5880cc190 2024-08-14T06:52:16.368 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.366+0000 7fb58f214640 1 -- 172.21.15.191:0/2096176261 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb588151b40 con 0x7fb5880cc190 2024-08-14T06:52:16.370 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.369+0000 7fb5857fa640 1 -- 172.21.15.191:0/2096176261 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7fb57c021070 con 0x7fb5880cc190 2024-08-14T06:52:16.444 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.443+0000 7fb58f214640 1 -- 172.21.15.191:0/2096176261 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "osd new", "uuid": "aa26d769-fd1e-4963-af87-a7006f935553", "id": 3} v 0) -- 0x7fb588074aa0 con 0x7fb5880cc190 2024-08-14T06:52:16.449 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.448+0000 7fb5857fa640 1 -- 172.21.15.191:0/2096176261 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "aa26d769-fd1e-4963-af87-a7006f935553", "id": 3}]=0 v6) ==== 112+0+2 (secure 0 0 0) 0x7fb57c011590 con 0x7fb5880cc190 2024-08-14T06:52:16.449 INFO:teuthology.orchestra.run.smithi191.stdout:3 2024-08-14T06:52:16.450 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.449+0000 7fb58f214640 1 -- 172.21.15.191:0/2096176261 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb5880cc190 msgr2=0x7fb588150250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.450 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.449+0000 7fb58f214640 1 --2- 172.21.15.191:0/2096176261 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb5880cc190 0x7fb588150250 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fb57c00c3f0 tx=0x7fb57c00c8b0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.450 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.449+0000 7fb58f214640 1 -- 172.21.15.191:0/2096176261 shutdown_connections 2024-08-14T06:52:16.450 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.449+0000 7fb58f214640 1 --2- 172.21.15.191:0/2096176261 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb5880b58c0 0x7fb5881f5720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.450 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.449+0000 7fb58f214640 1 --2- 172.21.15.191:0/2096176261 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb588152080 0x7fb5880b5380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.450 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.449+0000 7fb58f214640 1 --2- 172.21.15.191:0/2096176261 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb5880cc190 0x7fb588150250 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.450 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.449+0000 7fb58f214640 1 -- 172.21.15.191:0/2096176261 >> 172.21.15.191:0/2096176261 conn(0x7fb5880b2b30 msgr2=0x7fb5880afe20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:16.451 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.449+0000 7fb58f214640 1 -- 172.21.15.191:0/2096176261 shutdown_connections 2024-08-14T06:52:16.451 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.449+0000 7fb58f214640 1 -- 172.21.15.191:0/2096176261 wait complete. 2024-08-14T06:52:16.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new 0cd7a5fd-20b0-40f2-9e45-fd71ceadb6e3 4 2024-08-14T06:52:16.576 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.575+0000 7faf357f0640 1 -- 172.21.15.191:0/465753284 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faf300cc190 msgr2=0x7faf30073790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.576 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.575+0000 7faf357f0640 1 --2- 172.21.15.191:0/465753284 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faf300cc190 0x7faf30073790 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7faf3006a680 tx=0x7faf2002a440 comp rx=0 tx=0).stop 2024-08-14T06:52:16.576 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.575+0000 7faf357f0640 1 -- 172.21.15.191:0/465753284 shutdown_connections 2024-08-14T06:52:16.576 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.575+0000 7faf357f0640 1 --2- 172.21.15.191:0/465753284 >> v2:172.21.15.191:3300/0 conn(0x7faf30073cd0 0x7faf30151b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.576 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.575+0000 7faf357f0640 1 --2- 172.21.15.191:0/465753284 >> v2:172.21.15.98:3300/0 conn(0x7faf30152080 0x7faf3015bb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.576 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.575+0000 7faf357f0640 1 --2- 172.21.15.191:0/465753284 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faf300cc190 0x7faf30073790 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.576 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.575+0000 7faf357f0640 1 -- 172.21.15.191:0/465753284 >> 172.21.15.191:0/465753284 conn(0x7faf300b2b30 msgr2=0x7faf300b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:16.576 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.575+0000 7faf357f0640 1 -- 172.21.15.191:0/465753284 shutdown_connections 2024-08-14T06:52:16.576 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.575+0000 7faf357f0640 1 -- 172.21.15.191:0/465753284 wait complete. 2024-08-14T06:52:16.577 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.577+0000 7faf357f0640 1 Processor -- start 2024-08-14T06:52:16.578 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.577+0000 7faf357f0640 1 -- start start 2024-08-14T06:52:16.580 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf357f0640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faf30073cd0 0x7faf301f4710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.580 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf357f0640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faf300cc190 0x7faf3006d4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf357f0640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faf30152080 0x7faf300c71f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf357f0640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7faf3015c1c0 con 0x7faf30152080 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf357f0640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7faf3015c040 con 0x7faf30073cd0 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf357f0640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7faf3015c340 con 0x7faf300cc190 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf27fff640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faf300cc190 0x7faf3006d4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf2effd640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faf30073cd0 0x7faf301f4710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf27fff640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faf300cc190 0x7faf3006d4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3301/0 says I am v2:172.21.15.191:37722/0 (socket says 172.21.15.191:37722) 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf27fff640 1 -- 172.21.15.191:0/3000776882 learned_addr learned my addr 172.21.15.191:0/3000776882 (peer_addr_for_me v2:172.21.15.191:0/0) 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf2f7fe640 1 --2- 172.21.15.191:0/3000776882 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faf30152080 0x7faf300c71f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf2effd640 1 -- 172.21.15.191:0/3000776882 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faf300cc190 msgr2=0x7faf3006d4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf2effd640 1 --2- 172.21.15.191:0/3000776882 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faf300cc190 0x7faf3006d4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf2effd640 1 -- 172.21.15.191:0/3000776882 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faf30152080 msgr2=0x7faf300c71f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf2effd640 1 --2- 172.21.15.191:0/3000776882 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faf30152080 0x7faf300c71f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf2effd640 1 -- 172.21.15.191:0/3000776882 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faf300c6a50 con 0x7faf30073cd0 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.578+0000 7faf27fff640 1 --2- 172.21.15.191:0/3000776882 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faf300cc190 0x7faf3006d4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.579+0000 7faf2effd640 1 --2- 172.21.15.191:0/3000776882 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faf30073cd0 0x7faf301f4710 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7faf20031000 tx=0x7faf200038a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.579+0000 7faf277fe640 1 -- 172.21.15.191:0/3000776882 <== mon.1 v2:172.21.15.191:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf2003e030 con 0x7faf30073cd0 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.579+0000 7faf277fe640 1 -- 172.21.15.191:0/3000776882 <== mon.1 v2:172.21.15.191:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7faf20003ca0 con 0x7faf30073cd0 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.579+0000 7faf357f0640 1 -- 172.21.15.191:0/3000776882 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faf301f7260 con 0x7faf30073cd0 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.579+0000 7faf277fe640 1 -- 172.21.15.191:0/3000776882 <== mon.1 v2:172.21.15.191:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf20035040 con 0x7faf30073cd0 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.579+0000 7faf357f0640 1 -- 172.21.15.191:0/3000776882 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7faf301f7820 con 0x7faf30073cd0 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.579+0000 7faf277fe640 1 -- 172.21.15.191:0/3000776882 <== mon.1 v2:172.21.15.191:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7faf20041030 con 0x7faf30073cd0 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.579+0000 7faf277fe640 1 -- 172.21.15.191:0/3000776882 <== mon.1 v2:172.21.15.191:3300/0 5 ==== osd_map(6..6 src has 1..6) ==== 1185+0+0 (secure 0 0 0) 0x7faf20043ba0 con 0x7faf30073cd0 2024-08-14T06:52:16.582 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.580+0000 7faf357f0640 1 -- 172.21.15.191:0/3000776882 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faf301f7a60 con 0x7faf30073cd0 2024-08-14T06:52:16.583 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.582+0000 7faf277fe640 1 -- 172.21.15.191:0/3000776882 <== mon.1 v2:172.21.15.191:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7faf301f7a60 con 0x7faf30073cd0 2024-08-14T06:52:16.656 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.655+0000 7faf357f0640 1 -- 172.21.15.191:0/3000776882 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_command({"prefix": "osd new", "uuid": "0cd7a5fd-20b0-40f2-9e45-fd71ceadb6e3", "id": 4} v 0) -- 0x7faf3006a2a0 con 0x7faf30073cd0 2024-08-14T06:52:16.660 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.659+0000 7faf277fe640 1 -- 172.21.15.191:0/3000776882 <== mon.1 v2:172.21.15.191:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "0cd7a5fd-20b0-40f2-9e45-fd71ceadb6e3", "id": 4}]=0 v7) ==== 112+0+2 (secure 0 0 0) 0x7faf3006a2a0 con 0x7faf30073cd0 2024-08-14T06:52:16.660 INFO:teuthology.orchestra.run.smithi191.stdout:4 2024-08-14T06:52:16.661 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.660+0000 7faf357f0640 1 -- 172.21.15.191:0/3000776882 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faf30073cd0 msgr2=0x7faf301f4710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.661 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.660+0000 7faf357f0640 1 --2- 172.21.15.191:0/3000776882 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faf30073cd0 0x7faf301f4710 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7faf20031000 tx=0x7faf200038a0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.661 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.660+0000 7faf357f0640 1 -- 172.21.15.191:0/3000776882 shutdown_connections 2024-08-14T06:52:16.661 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.660+0000 7faf357f0640 1 --2- 172.21.15.191:0/3000776882 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7faf30152080 0x7faf300c71f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.662 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.660+0000 7faf357f0640 1 --2- 172.21.15.191:0/3000776882 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7faf300cc190 0x7faf3006d4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.662 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.660+0000 7faf357f0640 1 --2- 172.21.15.191:0/3000776882 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7faf30073cd0 0x7faf301f4710 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.662 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.660+0000 7faf357f0640 1 -- 172.21.15.191:0/3000776882 >> 172.21.15.191:0/3000776882 conn(0x7faf300b2b30 msgr2=0x7faf30151440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:16.662 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.660+0000 7faf357f0640 1 -- 172.21.15.191:0/3000776882 shutdown_connections 2024-08-14T06:52:16.662 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.660+0000 7faf357f0640 1 -- 172.21.15.191:0/3000776882 wait complete. 2024-08-14T06:52:16.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new fbc7b048-ee2c-46e1-81e9-854d0f132434 5 2024-08-14T06:52:16.788 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.787+0000 7fdce59d5640 1 -- 172.21.15.191:0/2285424784 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdce0072bf0 msgr2=0x7fdce0073000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.789 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.787+0000 7fdce59d5640 1 --2- 172.21.15.191:0/2285424784 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdce0072bf0 0x7fdce0073000 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fdcd8005fd0 tx=0x7fdcd802a9c0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.789 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.787+0000 7fdce59d5640 1 -- 172.21.15.191:0/2285424784 shutdown_connections 2024-08-14T06:52:16.789 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.787+0000 7fdce59d5640 1 --2- 172.21.15.191:0/2285424784 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdce0072bf0 0x7fdce0073000 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.789 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.787+0000 7fdce59d5640 1 --2- 172.21.15.191:0/2285424784 >> v2:172.21.15.98:3300/0 conn(0x7fdce00cde80 0x7fdce00ae530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.789 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.787+0000 7fdce59d5640 1 --2- 172.21.15.191:0/2285424784 >> v2:172.21.15.191:3300/0 conn(0x7fdce00cda70 0x7fdce00adff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.789 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.787+0000 7fdce59d5640 1 -- 172.21.15.191:0/2285424784 >> 172.21.15.191:0/2285424784 conn(0x7fdce00cfea0 msgr2=0x7fdce00d02b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:16.789 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.787+0000 7fdce59d5640 1 -- 172.21.15.191:0/2285424784 shutdown_connections 2024-08-14T06:52:16.789 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.787+0000 7fdce59d5640 1 -- 172.21.15.191:0/2285424784 wait complete. 2024-08-14T06:52:16.789 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.789+0000 7fdce59d5640 1 Processor -- start 2024-08-14T06:52:16.790 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.789+0000 7fdce59d5640 1 -- start start 2024-08-14T06:52:16.793 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.790+0000 7fdce59d5640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdce0072bf0 0x7fdce016c8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.793 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.790+0000 7fdce59d5640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fdce00cda70 0x7fdce00b4f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.793 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.790+0000 7fdce59d5640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fdce00cde80 0x7fdce00b54b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:16.793 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.790+0000 7fdce59d5640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7fdce00cae70 con 0x7fdce00cde80 2024-08-14T06:52:16.793 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.790+0000 7fdce59d5640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7fdce00cacf0 con 0x7fdce00cda70 2024-08-14T06:52:16.793 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.790+0000 7fdce59d5640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7fdce00caff0 con 0x7fdce0072bf0 2024-08-14T06:52:16.793 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcdeffd640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdce0072bf0 0x7fdce016c8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.793 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcde7fc640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fdce00cda70 0x7fdce00b4f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.793 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcde7fc640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fdce00cda70 0x7fdce00b4f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3300/0 says I am v2:172.21.15.191:58584/0 (socket says 172.21.15.191:58584) 2024-08-14T06:52:16.793 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcde7fc640 1 -- 172.21.15.191:0/4198707796 learned_addr learned my addr 172.21.15.191:0/4198707796 (peer_addr_for_me v2:172.21.15.191:0/0) 2024-08-14T06:52:16.794 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcdf7fe640 1 --2- 172.21.15.191:0/4198707796 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fdce00cde80 0x7fdce00b54b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:16.794 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcdeffd640 1 -- 172.21.15.191:0/4198707796 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fdce00cda70 msgr2=0x7fdce00b4f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.794 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcdeffd640 1 --2- 172.21.15.191:0/4198707796 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fdce00cda70 0x7fdce00b4f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.794 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcdeffd640 1 -- 172.21.15.191:0/4198707796 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fdce00cde80 msgr2=0x7fdce00b54b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.794 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcdeffd640 1 --2- 172.21.15.191:0/4198707796 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fdce00cde80 0x7fdce00b54b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.794 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcdeffd640 1 -- 172.21.15.191:0/4198707796 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdce00c7a50 con 0x7fdce0072bf0 2024-08-14T06:52:16.794 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcdeffd640 1 --2- 172.21.15.191:0/4198707796 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdce0072bf0 0x7fdce016c8b0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fdcd000e680 tx=0x7fdcd000eb40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:16.794 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcc7fff640 1 -- 172.21.15.191:0/4198707796 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdcd000cab0 con 0x7fdce0072bf0 2024-08-14T06:52:16.794 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcc7fff640 1 -- 172.21.15.191:0/4198707796 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fdcd000cd40 con 0x7fdce0072bf0 2024-08-14T06:52:16.795 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdce59d5640 1 -- 172.21.15.191:0/4198707796 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdce0174950 con 0x7fdce0072bf0 2024-08-14T06:52:16.795 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdcc7fff640 1 -- 172.21.15.191:0/4198707796 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdcd0010690 con 0x7fdce0072bf0 2024-08-14T06:52:16.795 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.791+0000 7fdce59d5640 1 -- 172.21.15.191:0/4198707796 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fdce01764b0 con 0x7fdce0072bf0 2024-08-14T06:52:16.795 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.792+0000 7fdcc7fff640 1 -- 172.21.15.191:0/4198707796 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fdcd0010830 con 0x7fdce0072bf0 2024-08-14T06:52:16.795 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.792+0000 7fdcc7fff640 1 -- 172.21.15.191:0/4198707796 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(7..7 src has 1..7) ==== 1300+0+0 (secure 0 0 0) 0x7fdcd0018b00 con 0x7fdce0072bf0 2024-08-14T06:52:16.795 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.792+0000 7fdce59d5640 1 -- 172.21.15.191:0/4198707796 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdce0176710 con 0x7fdce0072bf0 2024-08-14T06:52:16.797 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.796+0000 7fdcc7fff640 1 -- 172.21.15.191:0/4198707796 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7fdcd0018da0 con 0x7fdce0072bf0 2024-08-14T06:52:16.872 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.871+0000 7fdce59d5640 1 -- 172.21.15.191:0/4198707796 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "osd new", "uuid": "fbc7b048-ee2c-46e1-81e9-854d0f132434", "id": 5} v 0) -- 0x7fdce0073000 con 0x7fdce0072bf0 2024-08-14T06:52:16.877 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.876+0000 7fdcc7fff640 1 -- 172.21.15.191:0/4198707796 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "fbc7b048-ee2c-46e1-81e9-854d0f132434", "id": 5}]=0 v8) ==== 112+0+2 (secure 0 0 0) 0x7fdcd0010c60 con 0x7fdce0072bf0 2024-08-14T06:52:16.877 INFO:teuthology.orchestra.run.smithi191.stdout:5 2024-08-14T06:52:16.878 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.877+0000 7fdce59d5640 1 -- 172.21.15.191:0/4198707796 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdce0072bf0 msgr2=0x7fdce016c8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:16.878 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.877+0000 7fdce59d5640 1 --2- 172.21.15.191:0/4198707796 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdce0072bf0 0x7fdce016c8b0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fdcd000e680 tx=0x7fdcd000eb40 comp rx=0 tx=0).stop 2024-08-14T06:52:16.878 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.877+0000 7fdce59d5640 1 -- 172.21.15.191:0/4198707796 shutdown_connections 2024-08-14T06:52:16.878 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.877+0000 7fdce59d5640 1 --2- 172.21.15.191:0/4198707796 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fdce00cde80 0x7fdce00b54b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.878 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.877+0000 7fdce59d5640 1 --2- 172.21.15.191:0/4198707796 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fdce00cda70 0x7fdce00b4f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.879 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.877+0000 7fdce59d5640 1 --2- 172.21.15.191:0/4198707796 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fdce0072bf0 0x7fdce016c8b0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:16.879 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.877+0000 7fdce59d5640 1 -- 172.21.15.191:0/4198707796 >> 172.21.15.191:0/4198707796 conn(0x7fdce00cfea0 msgr2=0x7fdce00b0010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:16.879 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.877+0000 7fdce59d5640 1 -- 172.21.15.191:0/4198707796 shutdown_connections 2024-08-14T06:52:16.879 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:16.877+0000 7fdce59d5640 1 -- 172.21.15.191:0/4198707796 wait complete. 2024-08-14T06:52:16.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new 3f5d0d02-eae3-4d2b-b611-03cbec6590a4 6 2024-08-14T06:52:17.003 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.001+0000 7f769c5d5640 1 -- 172.21.15.191:0/3812229259 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f769409ad00 msgr2=0x7f76940d0210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.003 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.001+0000 7f769c5d5640 1 --2- 172.21.15.191:0/3812229259 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f769409ad00 0x7f76940d0210 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f7694074980 tx=0x7f768402a560 comp rx=0 tx=0).stop 2024-08-14T06:52:17.003 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.001+0000 7f769c5d5640 1 -- 172.21.15.191:0/3812229259 shutdown_connections 2024-08-14T06:52:17.003 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.001+0000 7f769c5d5640 1 --2- 172.21.15.191:0/3812229259 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f769409ad00 0x7f76940d0210 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.003 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.001+0000 7f769c5d5640 1 --2- 172.21.15.191:0/3812229259 >> v2:172.21.15.98:3300/0 conn(0x7f76940b4690 0x7f76940b30d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.003 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.001+0000 7f769c5d5640 1 -- 172.21.15.191:0/3812229259 >> 172.21.15.191:0/3812229259 conn(0x7f76940b3b70 msgr2=0x7f76940b3f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:17.003 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.002+0000 7f769c5d5640 1 -- 172.21.15.191:0/3812229259 shutdown_connections 2024-08-14T06:52:17.003 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.002+0000 7f769c5d5640 1 -- 172.21.15.191:0/3812229259 wait complete. 2024-08-14T06:52:17.004 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.003+0000 7f769c5d5640 1 Processor -- start 2024-08-14T06:52:17.004 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.003+0000 7f769c5d5640 1 -- start start 2024-08-14T06:52:17.007 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769c5d5640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f769409ad00 0x7f7694158570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:17.007 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769c5d5640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f769409b110 0x7f76941528b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:17.007 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769c5d5640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f76940b4690 0x7f7694154d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:17.007 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769c5d5640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f76940b51a0 con 0x7f769409b110 2024-08-14T06:52:17.007 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769c5d5640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f76940b5020 con 0x7f76940b4690 2024-08-14T06:52:17.007 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769c5d5640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f76940b5320 con 0x7f769409ad00 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769ab72640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f76940b4690 0x7f7694154d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769a371640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f769409ad00 0x7f7694158570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769ab72640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f76940b4690 0x7f7694154d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3300/0 says I am v2:172.21.15.191:58606/0 (socket says 172.21.15.191:58606) 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769a371640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f769409ad00 0x7f7694158570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3301/0 says I am v2:172.21.15.191:37768/0 (socket says 172.21.15.191:37768) 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769ab72640 1 -- 172.21.15.191:0/512798745 learned_addr learned my addr 172.21.15.191:0/512798745 (peer_addr_for_me v2:172.21.15.191:0/0) 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f7699b70640 1 --2- 172.21.15.191:0/512798745 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f769409b110 0x7f76941528b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769a371640 1 -- 172.21.15.191:0/512798745 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f76940b4690 msgr2=0x7f7694154d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769a371640 1 --2- 172.21.15.191:0/512798745 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f76940b4690 0x7f7694154d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769a371640 1 -- 172.21.15.191:0/512798745 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f769409b110 msgr2=0x7f76941528b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769a371640 1 --2- 172.21.15.191:0/512798745 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f769409b110 0x7f76941528b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769a371640 1 -- 172.21.15.191:0/512798745 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7694158ab0 con 0x7f769409ad00 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769ab72640 1 --2- 172.21.15.191:0/512798745 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f76940b4690 0x7f7694154d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:17.008 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f7699b70640 1 --2- 172.21.15.191:0/512798745 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f769409b110 0x7f76941528b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:17.009 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769a371640 1 --2- 172.21.15.191:0/512798745 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f769409ad00 0x7f7694158570 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f769000ea20 tx=0x7f769000eee0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:17.009 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f768affd640 1 -- 172.21.15.191:0/512798745 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f769000cd80 con 0x7f769409ad00 2024-08-14T06:52:17.009 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f768affd640 1 -- 172.21.15.191:0/512798745 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7690010430 con 0x7f769409ad00 2024-08-14T06:52:17.009 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f769c5d5640 1 -- 172.21.15.191:0/512798745 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7694152df0 con 0x7f769409ad00 2024-08-14T06:52:17.009 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.005+0000 7f768affd640 1 -- 172.21.15.191:0/512798745 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7690010840 con 0x7f769409ad00 2024-08-14T06:52:17.009 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.006+0000 7f769c5d5640 1 -- 172.21.15.191:0/512798745 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f7694157a90 con 0x7f769409ad00 2024-08-14T06:52:17.009 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.006+0000 7f768affd640 1 -- 172.21.15.191:0/512798745 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f76900109e0 con 0x7f769409ad00 2024-08-14T06:52:17.009 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.006+0000 7f768affd640 1 -- 172.21.15.191:0/512798745 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(8..8 src has 1..8) ==== 1415+0+0 (secure 0 0 0) 0x7f7690018d80 con 0x7f769409ad00 2024-08-14T06:52:17.009 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.006+0000 7f769c5d5640 1 -- 172.21.15.191:0/512798745 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f76940b30d0 con 0x7f769409ad00 2024-08-14T06:52:17.011 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.010+0000 7f768affd640 1 -- 172.21.15.191:0/512798745 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f76900213f0 con 0x7f769409ad00 2024-08-14T06:52:17.087 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.086+0000 7f769c5d5640 1 -- 172.21.15.191:0/512798745 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "osd new", "uuid": "3f5d0d02-eae3-4d2b-b611-03cbec6590a4", "id": 6} v 0) -- 0x7f769415eea0 con 0x7f769409ad00 2024-08-14T06:52:17.090 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.090+0000 7f768affd640 1 -- 172.21.15.191:0/512798745 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "3f5d0d02-eae3-4d2b-b611-03cbec6590a4", "id": 6}]=0 v9) ==== 112+0+2 (secure 0 0 0) 0x7f76900215d0 con 0x7f769409ad00 2024-08-14T06:52:17.091 INFO:teuthology.orchestra.run.smithi191.stdout:6 2024-08-14T06:52:17.092 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.091+0000 7f769c5d5640 1 -- 172.21.15.191:0/512798745 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f769409ad00 msgr2=0x7f7694158570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.092 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.091+0000 7f769c5d5640 1 --2- 172.21.15.191:0/512798745 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f769409ad00 0x7f7694158570 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f769000ea20 tx=0x7f769000eee0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.092 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.091+0000 7f769c5d5640 1 -- 172.21.15.191:0/512798745 shutdown_connections 2024-08-14T06:52:17.092 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.091+0000 7f769c5d5640 1 --2- 172.21.15.191:0/512798745 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f76940b4690 0x7f7694154d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.092 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.091+0000 7f769c5d5640 1 --2- 172.21.15.191:0/512798745 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f769409b110 0x7f76941528b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.092 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.091+0000 7f769c5d5640 1 --2- 172.21.15.191:0/512798745 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f769409ad00 0x7f7694158570 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.093 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.091+0000 7f769c5d5640 1 -- 172.21.15.191:0/512798745 >> 172.21.15.191:0/512798745 conn(0x7f76940b3b70 msgr2=0x7f76940b6800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:17.093 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.091+0000 7f769c5d5640 1 -- 172.21.15.191:0/512798745 shutdown_connections 2024-08-14T06:52:17.093 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.091+0000 7f769c5d5640 1 -- 172.21.15.191:0/512798745 wait complete. 2024-08-14T06:52:17.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new 8d15135a-d5ad-47b1-988a-c01ccd36426e 7 2024-08-14T06:52:17.217 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.216+0000 7f9bfb33f640 1 -- 172.21.15.191:0/716791664 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9bf4073cd0 msgr2=0x7f9bf4151b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.218 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.216+0000 7f9bfb33f640 1 --2- 172.21.15.191:0/716791664 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9bf4073cd0 0x7f9bf4151b40 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f9bf0005fd0 tx=0x7f9bf002a4b0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.218 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.216+0000 7f9bfb33f640 1 -- 172.21.15.191:0/716791664 shutdown_connections 2024-08-14T06:52:17.218 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.216+0000 7f9bfb33f640 1 --2- 172.21.15.191:0/716791664 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9bf4073cd0 0x7f9bf4151b40 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.218 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.216+0000 7f9bfb33f640 1 --2- 172.21.15.191:0/716791664 >> v2:172.21.15.191:3300/0 conn(0x7f9bf40cc190 0x7f9bf4073790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.218 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.216+0000 7f9bfb33f640 1 -- 172.21.15.191:0/716791664 >> 172.21.15.191:0/716791664 conn(0x7f9bf40b2b30 msgr2=0x7f9bf40b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:17.218 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.216+0000 7f9bfb33f640 1 -- 172.21.15.191:0/716791664 shutdown_connections 2024-08-14T06:52:17.218 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.216+0000 7f9bfb33f640 1 -- 172.21.15.191:0/716791664 wait complete. 2024-08-14T06:52:17.219 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.218+0000 7f9bfb33f640 1 Processor -- start 2024-08-14T06:52:17.219 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.218+0000 7f9bfb33f640 1 -- start start 2024-08-14T06:52:17.222 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.219+0000 7f9bfb33f640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9bf4073cd0 0x7f9bf41f2f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:17.222 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.219+0000 7f9bfb33f640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9bf40cc190 0x7f9bf414f4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:17.222 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.219+0000 7f9bfb33f640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9bf4152080 0x7f9bf414fa10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:17.222 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.219+0000 7f9bfb33f640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f9bf415c5a0 con 0x7f9bf4152080 2024-08-14T06:52:17.222 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.219+0000 7f9bfb33f640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f9bf415c420 con 0x7f9bf4073cd0 2024-08-14T06:52:17.222 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.219+0000 7f9bfb33f640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f9bf415c720 con 0x7f9bf40cc190 2024-08-14T06:52:17.222 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.219+0000 7f9bf88da640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9bf40cc190 0x7f9bf414f4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:17.222 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.219+0000 7f9bf88da640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9bf40cc190 0x7f9bf414f4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.191:3301/0 says I am v2:172.21.15.191:37792/0 (socket says 172.21.15.191:37792) 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.219+0000 7f9bf90db640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9bf4073cd0 0x7f9bf41f2f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.219+0000 7f9bf88da640 1 -- 172.21.15.191:0/1481338671 learned_addr learned my addr 172.21.15.191:0/1481338671 (peer_addr_for_me v2:172.21.15.191:0/0) 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bf98dc640 1 --2- 172.21.15.191:0/1481338671 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9bf4152080 0x7f9bf414fa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bf88da640 1 -- 172.21.15.191:0/1481338671 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9bf4073cd0 msgr2=0x7f9bf41f2f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bf88da640 1 --2- 172.21.15.191:0/1481338671 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9bf4073cd0 0x7f9bf41f2f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bf88da640 1 -- 172.21.15.191:0/1481338671 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9bf4152080 msgr2=0x7f9bf414fa10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bf88da640 1 --2- 172.21.15.191:0/1481338671 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9bf4152080 0x7f9bf414fa10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bf88da640 1 -- 172.21.15.191:0/1481338671 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9bf41f6f10 con 0x7f9bf40cc190 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bf98dc640 1 --2- 172.21.15.191:0/1481338671 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9bf4152080 0x7f9bf414fa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bf90db640 1 --2- 172.21.15.191:0/1481338671 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9bf4073cd0 0x7f9bf41f2f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bf88da640 1 --2- 172.21.15.191:0/1481338671 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9bf40cc190 0x7f9bf414f4d0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f9bf00099f0 tx=0x7f9bf00038a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9be9ffb640 1 -- 172.21.15.191:0/1481338671 <== mon.2 v2:172.21.15.191:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9bf003e030 con 0x7f9bf40cc190 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9be9ffb640 1 -- 172.21.15.191:0/1481338671 <== mon.2 v2:172.21.15.191:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9bf0003db0 con 0x7f9bf40cc190 2024-08-14T06:52:17.223 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9be9ffb640 1 -- 172.21.15.191:0/1481338671 <== mon.2 v2:172.21.15.191:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9bf0035040 con 0x7f9bf40cc190 2024-08-14T06:52:17.224 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bfb33f640 1 -- 172.21.15.191:0/1481338671 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9bf41f71f0 con 0x7f9bf40cc190 2024-08-14T06:52:17.224 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.220+0000 7f9bfb33f640 1 -- 172.21.15.191:0/1481338671 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f9bf41f77b0 con 0x7f9bf40cc190 2024-08-14T06:52:17.224 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.221+0000 7f9be9ffb640 1 -- 172.21.15.191:0/1481338671 <== mon.2 v2:172.21.15.191:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f9bf0041030 con 0x7f9bf40cc190 2024-08-14T06:52:17.224 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.221+0000 7f9be9ffb640 1 -- 172.21.15.191:0/1481338671 <== mon.2 v2:172.21.15.191:3301/0 5 ==== osd_map(9..9 src has 1..9) ==== 1530+0+0 (secure 0 0 0) 0x7f9bf0031400 con 0x7f9bf40cc190 2024-08-14T06:52:17.224 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.221+0000 7f9bfb33f640 1 -- 172.21.15.191:0/1481338671 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9bf4152aa0 con 0x7f9bf40cc190 2024-08-14T06:52:17.224 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.224+0000 7f9be9ffb640 1 -- 172.21.15.191:0/1481338671 <== mon.2 v2:172.21.15.191:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f9bf0004f20 con 0x7f9bf40cc190 2024-08-14T06:52:17.299 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.298+0000 7f9bfb33f640 1 -- 172.21.15.191:0/1481338671 --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_command({"prefix": "osd new", "uuid": "8d15135a-d5ad-47b1-988a-c01ccd36426e", "id": 7} v 0) -- 0x7f9bf4151b40 con 0x7f9bf40cc190 2024-08-14T06:52:17.303 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.302+0000 7f9be9ffb640 1 -- 172.21.15.191:0/1481338671 <== mon.2 v2:172.21.15.191:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "8d15135a-d5ad-47b1-988a-c01ccd36426e", "id": 7}]=0 v10) ==== 112+0+2 (secure 0 0 0) 0x7f9bf00437d0 con 0x7f9bf40cc190 2024-08-14T06:52:17.303 INFO:teuthology.orchestra.run.smithi191.stdout:7 2024-08-14T06:52:17.304 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.303+0000 7f9bfb33f640 1 -- 172.21.15.191:0/1481338671 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9bf40cc190 msgr2=0x7f9bf414f4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.304 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.303+0000 7f9bfb33f640 1 --2- 172.21.15.191:0/1481338671 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9bf40cc190 0x7f9bf414f4d0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f9bf00099f0 tx=0x7f9bf00038a0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.304 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.303+0000 7f9bfb33f640 1 -- 172.21.15.191:0/1481338671 shutdown_connections 2024-08-14T06:52:17.305 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.303+0000 7f9bfb33f640 1 --2- 172.21.15.191:0/1481338671 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9bf4152080 0x7f9bf414fa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.305 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.303+0000 7f9bfb33f640 1 --2- 172.21.15.191:0/1481338671 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9bf40cc190 0x7f9bf414f4d0 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.305 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.303+0000 7f9bfb33f640 1 --2- 172.21.15.191:0/1481338671 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9bf4073cd0 0x7f9bf41f2f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.305 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.303+0000 7f9bfb33f640 1 -- 172.21.15.191:0/1481338671 >> 172.21.15.191:0/1481338671 conn(0x7f9bf40b2b30 msgr2=0x7f9bf40b22e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:17.305 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.303+0000 7f9bfb33f640 1 -- 172.21.15.191:0/1481338671 shutdown_connections 2024-08-14T06:52:17.305 INFO:teuthology.orchestra.run.smithi191.stderr:2024-08-14T06:52:17.303+0000 7f9bfb33f640 1 -- 172.21.15.191:0/1481338671 wait complete. 2024-08-14T06:52:17.315 INFO:tasks.ceph.osd.0:Restarting daemon 2024-08-14T06:52:17.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-08-14T06:52:17.317 INFO:tasks.ceph.osd.0:Started 2024-08-14T06:52:17.317 INFO:tasks.ceph.osd.1:Restarting daemon 2024-08-14T06:52:17.318 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-08-14T06:52:17.319 INFO:tasks.ceph.osd.1:Started 2024-08-14T06:52:17.320 INFO:tasks.ceph.osd.2:Restarting daemon 2024-08-14T06:52:17.320 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-08-14T06:52:17.322 INFO:tasks.ceph.osd.2:Started 2024-08-14T06:52:17.322 INFO:tasks.ceph.osd.3:Restarting daemon 2024-08-14T06:52:17.322 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-08-14T06:52:17.324 INFO:tasks.ceph.osd.3:Started 2024-08-14T06:52:17.324 INFO:tasks.ceph.osd.4:Restarting daemon 2024-08-14T06:52:17.324 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 4 2024-08-14T06:52:17.326 INFO:tasks.ceph.osd.4:Started 2024-08-14T06:52:17.327 INFO:tasks.ceph.osd.5:Restarting daemon 2024-08-14T06:52:17.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 5 2024-08-14T06:52:17.329 INFO:tasks.ceph.osd.5:Started 2024-08-14T06:52:17.329 INFO:tasks.ceph.osd.6:Restarting daemon 2024-08-14T06:52:17.329 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 6 2024-08-14T06:52:17.331 INFO:tasks.ceph.osd.6:Started 2024-08-14T06:52:17.331 INFO:tasks.ceph.osd.7:Restarting daemon 2024-08-14T06:52:17.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 7 2024-08-14T06:52:17.333 INFO:tasks.ceph.osd.7:Started 2024-08-14T06:52:17.334 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-08-14T06:52:17.476 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.474+0000 7f9eea776640 1 -- 172.21.15.98:0/2573545445 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9ee40b30d0 msgr2=0x7f9ee4160720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.477 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.474+0000 7f9eea776640 1 --2- 172.21.15.98:0/2573545445 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9ee40b30d0 0x7f9ee4160720 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f9ee4073cc0 tx=0x7f9ed802a560 comp rx=0 tx=0).stop 2024-08-14T06:52:17.477 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.474+0000 7f9eea776640 1 -- 172.21.15.98:0/2573545445 shutdown_connections 2024-08-14T06:52:17.477 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.474+0000 7f9eea776640 1 --2- 172.21.15.98:0/2573545445 >> v2:172.21.15.191:3300/0 conn(0x7f9ee4160c60 0x7f9ee4161070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.477 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.474+0000 7f9eea776640 1 --2- 172.21.15.98:0/2573545445 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9ee40b30d0 0x7f9ee4160720 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.477 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.474+0000 7f9eea776640 1 -- 172.21.15.98:0/2573545445 >> 172.21.15.98:0/2573545445 conn(0x7f9ee40b3b70 msgr2=0x7f9ee40b3f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:17.477 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.474+0000 7f9eea776640 1 -- 172.21.15.98:0/2573545445 shutdown_connections 2024-08-14T06:52:17.477 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.474+0000 7f9eea776640 1 -- 172.21.15.98:0/2573545445 wait complete. 2024-08-14T06:52:17.478 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.476+0000 7f9eea776640 1 Processor -- start 2024-08-14T06:52:17.478 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.476+0000 7f9eea776640 1 -- start start 2024-08-14T06:52:17.481 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9eea776640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9ee40b30d0 0x7f9ee41e8f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:17.481 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9eea776640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9ee40b4690 0x7f9ee4157bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:17.481 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9eea776640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9ee4160c60 0x7f9ee415f380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:17.481 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9eea776640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f9ee40b51a0 con 0x7f9ee40b4690 2024-08-14T06:52:17.481 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9eea776640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f9ee40b5020 con 0x7f9ee4160c60 2024-08-14T06:52:17.481 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9eea776640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f9ee40b5320 con 0x7f9ee40b30d0 2024-08-14T06:52:17.481 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee37fe640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9ee40b4690 0x7f9ee4157bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:17.481 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee3fff640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9ee40b30d0 0x7f9ee41e8f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:17.481 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee8d13640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9ee4160c60 0x7f9ee415f380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee37fe640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9ee40b4690 0x7f9ee4157bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:38766/0 (socket says 172.21.15.98:38766) 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee37fe640 1 -- 172.21.15.98:0/1806305662 learned_addr learned my addr 172.21.15.98:0/1806305662 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee8d13640 1 -- 172.21.15.98:0/1806305662 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9ee40b30d0 msgr2=0x7f9ee41e8f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee8d13640 1 --2- 172.21.15.98:0/1806305662 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9ee40b30d0 0x7f9ee41e8f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee8d13640 1 -- 172.21.15.98:0/1806305662 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9ee40b4690 msgr2=0x7f9ee4157bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee8d13640 1 --2- 172.21.15.98:0/1806305662 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9ee40b4690 0x7f9ee4157bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee8d13640 1 -- 172.21.15.98:0/1806305662 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9ee409f620 con 0x7f9ee4160c60 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee3fff640 1 --2- 172.21.15.98:0/1806305662 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9ee40b30d0 0x7f9ee41e8f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee37fe640 1 --2- 172.21.15.98:0/1806305662 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9ee40b4690 0x7f9ee4157bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.478+0000 7f9ee8d13640 1 --2- 172.21.15.98:0/1806305662 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9ee4160c60 0x7f9ee415f380 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f9edc00e9b0 tx=0x7f9edc00ee70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.479+0000 7f9ee0ff9640 1 -- 172.21.15.98:0/1806305662 <== mon.1 v2:172.21.15.191:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9edc00cd80 con 0x7f9ee4160c60 2024-08-14T06:52:17.482 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.479+0000 7f9ee0ff9640 1 -- 172.21.15.98:0/1806305662 <== mon.1 v2:172.21.15.191:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9edc004c10 con 0x7f9ee4160c60 2024-08-14T06:52:17.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.479+0000 7f9ee0ff9640 1 -- 172.21.15.98:0/1806305662 <== mon.1 v2:172.21.15.191:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9edc005020 con 0x7f9ee4160c60 2024-08-14T06:52:17.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.479+0000 7f9eea776640 1 -- 172.21.15.98:0/1806305662 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9ee409f900 con 0x7f9ee4160c60 2024-08-14T06:52:17.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.479+0000 7f9eea776640 1 -- 172.21.15.98:0/1806305662 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9ee409fec0 con 0x7f9ee4160c60 2024-08-14T06:52:17.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.479+0000 7f9ee0ff9640 1 -- 172.21.15.98:0/1806305662 <== mon.1 v2:172.21.15.191:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f9edc005550 con 0x7f9ee4160c60 2024-08-14T06:52:17.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.479+0000 7f9ee0ff9640 1 -- 172.21.15.98:0/1806305662 <== mon.1 v2:172.21.15.191:3300/0 5 ==== osd_map(10..10 src has 1..10) ==== 1645+0+0 (secure 0 0 0) 0x7f9edc005840 con 0x7f9ee4160c60 2024-08-14T06:52:17.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.480+0000 7f9eea776640 1 -- 172.21.15.98:0/1806305662 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9ee409f380 con 0x7f9ee4160c60 2024-08-14T06:52:17.483 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.482+0000 7f9ee0ff9640 1 -- 172.21.15.98:0/1806305662 <== mon.1 v2:172.21.15.191:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f9edc038860 con 0x7f9ee4160c60 2024-08-14T06:52:17.546 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.545+0000 7f9eea776640 1 -- 172.21.15.98:0/1806305662 --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f9ee4161390 con 0x7f9ee4160c60 2024-08-14T06:52:17.547 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.546+0000 7f9ee0ff9640 1 -- 172.21.15.98:0/1806305662 <== mon.1 v2:172.21.15.191:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v10) ==== 74+0+7058 (secure 0 0 0) 0x7f9edc026020 con 0x7f9ee4160c60 2024-08-14T06:52:17.547 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:52:17.548 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":10,"fsid":"1a641e76-d282-43fb-b02c-28f3852c1ab1","created":"2024-08-14T06:52:14.033527+0000","modified":"2024-08-14T06:52:17.300583+0000","last_up_change":"0.000000","last_in_change":"2024-08-14T06:52:17.300583+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"273f37e4-1d4c-46cd-96bc-3d7fce969572","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"ebd9bc08-f436-4e26-a0ba-d79a2220ba5e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"9979a52a-0573-4019-a471-2ad384d88176","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"aa26d769-fd1e-4963-af87-a7006f935553","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"0cd7a5fd-20b0-40f2-9e45-fd71ceadb6e3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"fbc7b048-ee2c-46e1-81e9-854d0f132434","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"3f5d0d02-eae3-4d2b-b611-03cbec6590a4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"8d15135a-d5ad-47b1-988a-c01ccd36426e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-14T06:52:17.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.547+0000 7f9eea776640 1 -- 172.21.15.98:0/1806305662 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9ee4160c60 msgr2=0x7f9ee415f380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:17.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.547+0000 7f9eea776640 1 --2- 172.21.15.98:0/1806305662 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9ee4160c60 0x7f9ee415f380 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f9edc00e9b0 tx=0x7f9edc00ee70 comp rx=0 tx=0).stop 2024-08-14T06:52:17.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.547+0000 7f9eea776640 1 -- 172.21.15.98:0/1806305662 shutdown_connections 2024-08-14T06:52:17.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.547+0000 7f9eea776640 1 --2- 172.21.15.98:0/1806305662 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9ee4160c60 0x7f9ee415f380 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.547+0000 7f9eea776640 1 --2- 172.21.15.98:0/1806305662 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9ee40b4690 0x7f9ee4157bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.547+0000 7f9eea776640 1 --2- 172.21.15.98:0/1806305662 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9ee40b30d0 0x7f9ee41e8f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:17.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.547+0000 7f9eea776640 1 -- 172.21.15.98:0/1806305662 >> 172.21.15.98:0/1806305662 conn(0x7f9ee40b3b70 msgr2=0x7f9ee4070a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:17.549 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.547+0000 7f9eea776640 1 -- 172.21.15.98:0/1806305662 shutdown_connections 2024-08-14T06:52:17.550 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:17.547+0000 7f9eea776640 1 -- 172.21.15.98:0/1806305662 wait complete. 2024-08-14T06:52:17.560 INFO:tasks.ceph.ceph_manager.ceph:[] 2024-08-14T06:52:17.560 INFO:tasks.ceph:Waiting for OSDs to come up 2024-08-14T06:52:18.316 INFO:tasks.ceph.osd.7.smithi191.stderr:2024-08-14T06:52:18.315+0000 7f4da0bd6640 -1 Falling back to public interface 2024-08-14T06:52:18.367 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-08-14T06:52:18.477 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.475+0000 7f872b5ce640 1 -- 172.21.15.98:0/1860144208 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f872415eea0 msgr2=0x7f872415f2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:18.478 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.475+0000 7f872b5ce640 1 --2- 172.21.15.98:0/1860144208 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f872415eea0 0x7f872415f2b0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f8714009830 tx=0x7f871402a480 comp rx=0 tx=0).stop 2024-08-14T06:52:18.478 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.475+0000 7f872b5ce640 1 -- 172.21.15.98:0/1860144208 shutdown_connections 2024-08-14T06:52:18.478 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.475+0000 7f872b5ce640 1 --2- 172.21.15.98:0/1860144208 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f872415eea0 0x7f872415f2b0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:18.478 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.475+0000 7f872b5ce640 1 --2- 172.21.15.98:0/1860144208 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f87240d0080 0x7f87240afed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:18.478 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.475+0000 7f872b5ce640 1 -- 172.21.15.98:0/1860144208 >> 172.21.15.98:0/1860144208 conn(0x7f872406d650 msgr2=0x7f87240b25d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:18.478 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.475+0000 7f872b5ce640 1 -- 172.21.15.98:0/1860144208 shutdown_connections 2024-08-14T06:52:18.478 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.475+0000 7f872b5ce640 1 -- 172.21.15.98:0/1860144208 wait complete. 2024-08-14T06:52:18.480 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.478+0000 7f872b5ce640 1 Processor -- start 2024-08-14T06:52:18.480 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.478+0000 7f872b5ce640 1 -- start start 2024-08-14T06:52:18.483 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.480+0000 7f872b5ce640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f87240d0080 0x7f87241441b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.480+0000 7f872b5ce640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f872415b8b0 0x7f87241e4ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.480+0000 7f872b5ce640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f872415eea0 0x7f87241e5220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.480+0000 7f872b5ce640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f8724152760 con 0x7f872415eea0 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.480+0000 7f872b5ce640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f87241525e0 con 0x7f87240d0080 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.480+0000 7f872b5ce640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f872415c3d0 con 0x7f872415b8b0 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8729b6b640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f872415eea0 0x7f87241e5220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8729b6b640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f872415eea0 0x7f87241e5220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:38774/0 (socket says 172.21.15.98:38774) 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8729b6b640 1 -- 172.21.15.98:0/2370508982 learned_addr learned my addr 172.21.15.98:0/2370508982 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8728b69640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f872415b8b0 0x7f87241e4ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f872936a640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f87240d0080 0x7f87241441b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8729b6b640 1 -- 172.21.15.98:0/2370508982 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f872415b8b0 msgr2=0x7f87241e4ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:18.484 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8729b6b640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f872415b8b0 0x7f87241e4ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8729b6b640 1 -- 172.21.15.98:0/2370508982 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f87240d0080 msgr2=0x7f87241441b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8729b6b640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f87240d0080 0x7f87241441b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8729b6b640 1 -- 172.21.15.98:0/2370508982 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8724152980 con 0x7f872415eea0 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8728b69640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f872415b8b0 0x7f87241e4ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f872936a640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f87240d0080 0x7f87241441b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8729b6b640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f872415eea0 0x7f87241e5220 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f871c00e9b0 tx=0x7f871c00ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8719ffb640 1 -- 172.21.15.98:0/2370508982 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f871c015070 con 0x7f872415eea0 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8719ffb640 1 -- 172.21.15.98:0/2370508982 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f871c001aa0 con 0x7f872415eea0 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f8719ffb640 1 -- 172.21.15.98:0/2370508982 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f871c001ef0 con 0x7f872415eea0 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f872b5ce640 1 -- 172.21.15.98:0/2370508982 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8724152c60 con 0x7f872415eea0 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.481+0000 7f872b5ce640 1 -- 172.21.15.98:0/2370508982 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8724153220 con 0x7f872415eea0 2024-08-14T06:52:18.485 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.482+0000 7f8719ffb640 1 -- 172.21.15.98:0/2370508982 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 2) ==== 49591+0+0 (secure 0 0 0) 0x7f871c018070 con 0x7f872415eea0 2024-08-14T06:52:18.486 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.482+0000 7f872b5ce640 1 -- 172.21.15.98:0/2370508982 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8724076500 con 0x7f872415eea0 2024-08-14T06:52:18.486 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.483+0000 7f8719ffb640 1 -- 172.21.15.98:0/2370508982 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(10..10 src has 1..10) ==== 1645+0+0 (secure 0 0 0) 0x7f871c053600 con 0x7f872415eea0 2024-08-14T06:52:18.486 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.485+0000 7f8719ffb640 1 -- 172.21.15.98:0/2370508982 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+75856 (secure 0 0 0) 0x7f8724076500 con 0x7f872415eea0 2024-08-14T06:52:18.550 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.549+0000 7f872b5ce640 1 -- 172.21.15.98:0/2370508982 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f87240af970 con 0x7f872415eea0 2024-08-14T06:52:18.551 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.549+0000 7f8719ffb640 1 -- 172.21.15.98:0/2370508982 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v10) ==== 74+0+7058 (secure 0 0 0) 0x7f87240cdfc0 con 0x7f872415eea0 2024-08-14T06:52:18.551 INFO:teuthology.misc.health.smithi098.stdout: 2024-08-14T06:52:18.551 INFO:teuthology.misc.health.smithi098.stdout:{"epoch":10,"fsid":"1a641e76-d282-43fb-b02c-28f3852c1ab1","created":"2024-08-14T06:52:14.033527+0000","modified":"2024-08-14T06:52:17.300583+0000","last_up_change":"0.000000","last_in_change":"2024-08-14T06:52:17.300583+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"273f37e4-1d4c-46cd-96bc-3d7fce969572","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"ebd9bc08-f436-4e26-a0ba-d79a2220ba5e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"9979a52a-0573-4019-a471-2ad384d88176","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"aa26d769-fd1e-4963-af87-a7006f935553","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"0cd7a5fd-20b0-40f2-9e45-fd71ceadb6e3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"fbc7b048-ee2c-46e1-81e9-854d0f132434","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"3f5d0d02-eae3-4d2b-b611-03cbec6590a4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"8d15135a-d5ad-47b1-988a-c01ccd36426e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-14T06:52:18.553 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.550+0000 7f872b5ce640 1 -- 172.21.15.98:0/2370508982 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f872415eea0 msgr2=0x7f87241e5220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:18.553 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.550+0000 7f872b5ce640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f872415eea0 0x7f87241e5220 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f871c00e9b0 tx=0x7f871c00ee70 comp rx=0 tx=0).stop 2024-08-14T06:52:18.553 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.550+0000 7f872b5ce640 1 -- 172.21.15.98:0/2370508982 shutdown_connections 2024-08-14T06:52:18.553 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.550+0000 7f872b5ce640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f872415eea0 0x7f87241e5220 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:18.553 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.550+0000 7f872b5ce640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f872415b8b0 0x7f87241e4ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:18.553 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.550+0000 7f872b5ce640 1 --2- 172.21.15.98:0/2370508982 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f87240d0080 0x7f87241441b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:18.553 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.550+0000 7f872b5ce640 1 -- 172.21.15.98:0/2370508982 >> 172.21.15.98:0/2370508982 conn(0x7f872406d650 msgr2=0x7f872415bcc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:18.553 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.551+0000 7f872b5ce640 1 -- 172.21.15.98:0/2370508982 shutdown_connections 2024-08-14T06:52:18.553 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:18.551+0000 7f872b5ce640 1 -- 172.21.15.98:0/2370508982 wait complete. 2024-08-14T06:52:18.561 DEBUG:teuthology.misc:0 of 8 OSDs are up 2024-08-14T06:52:18.775 INFO:tasks.ceph.osd.6.smithi191.stderr:2024-08-14T06:52:18.774+0000 7f7d4620d640 -1 Falling back to public interface 2024-08-14T06:52:18.788 INFO:tasks.ceph.osd.4.smithi191.stderr:2024-08-14T06:52:18.787+0000 7fbc1bc2c640 -1 Falling back to public interface 2024-08-14T06:52:18.821 INFO:tasks.ceph.osd.5.smithi191.stderr:2024-08-14T06:52:18.820+0000 7f0131c0d640 -1 Falling back to public interface 2024-08-14T06:52:18.846 INFO:tasks.ceph.osd.3.smithi098.stderr:2024-08-14T06:52:18.844+0000 7f7134aff640 -1 Falling back to public interface 2024-08-14T06:52:19.106 INFO:tasks.ceph.osd.0.smithi098.stderr:2024-08-14T06:52:19.104+0000 7f6cedb88640 -1 Falling back to public interface 2024-08-14T06:52:19.334 INFO:tasks.ceph.osd.1.smithi098.stderr:2024-08-14T06:52:19.332+0000 7ffa8ee1b640 -1 Falling back to public interface 2024-08-14T06:52:19.336 INFO:tasks.ceph.osd.2.smithi098.stderr:2024-08-14T06:52:19.334+0000 7f83d140e640 -1 Falling back to public interface 2024-08-14T06:52:19.447 INFO:tasks.ceph.mgr.x.smithi191.stderr:2024-08-14T06:52:19.446+0000 7f81a1a29640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-08-14T06:52:19.447 INFO:tasks.ceph.mgr.x.smithi191.stderr:2024-08-14T06:52:19.447+0000 7f81a1a29640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-08-14T06:52:22.098 INFO:tasks.ceph.osd.7.smithi191.stderr:2024-08-14T06:52:22.097+0000 7f4da0bd6640 -1 osd.7 0 log_to_monitors true 2024-08-14T06:52:22.544 INFO:tasks.ceph.osd.6.smithi191.stderr:2024-08-14T06:52:22.544+0000 7f7d4620d640 -1 osd.6 0 log_to_monitors true 2024-08-14T06:52:22.603 INFO:tasks.ceph.osd.4.smithi191.stderr:2024-08-14T06:52:22.603+0000 7fbc1bc2c640 -1 osd.4 0 log_to_monitors true 2024-08-14T06:52:22.931 INFO:tasks.ceph.osd.0.smithi098.stderr:2024-08-14T06:52:22.930+0000 7f6cedb88640 -1 osd.0 0 log_to_monitors true 2024-08-14T06:52:23.120 INFO:tasks.ceph.osd.2.smithi098.stderr:2024-08-14T06:52:23.119+0000 7f83d140e640 -1 osd.2 0 log_to_monitors true 2024-08-14T06:52:23.125 INFO:tasks.ceph.osd.5.smithi191.stderr:2024-08-14T06:52:23.124+0000 7f0131c0d640 -1 osd.5 0 log_to_monitors true 2024-08-14T06:52:23.167 INFO:tasks.ceph.osd.1.smithi098.stderr:2024-08-14T06:52:23.165+0000 7ffa8ee1b640 -1 osd.1 0 log_to_monitors true 2024-08-14T06:52:23.408 INFO:tasks.ceph.osd.3.smithi098.stderr:2024-08-14T06:52:23.406+0000 7f7134aff640 -1 osd.3 0 log_to_monitors true 2024-08-14T06:52:24.341 INFO:tasks.ceph.osd.7.smithi191.stderr:2024-08-14T06:52:24.340+0000 7f4d9d380640 -1 osd.7 0 waiting for initial osdmap 2024-08-14T06:52:24.351 INFO:tasks.ceph.osd.7.smithi191.stderr:2024-08-14T06:52:24.350+0000 7f4d94961640 -1 osd.7 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-14T06:52:25.368 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-08-14T06:52:25.490 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.487+0000 7fe77e30d640 1 -- 172.21.15.98:0/2193128233 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fe778072890 msgr2=0x7fe778074620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:25.490 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.487+0000 7fe77e30d640 1 --2- 172.21.15.98:0/2193128233 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fe778072890 0x7fe778074620 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fe768005fd0 tx=0x7fe76802a530 comp rx=0 tx=0).stop 2024-08-14T06:52:25.490 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.488+0000 7fe77e30d640 1 -- 172.21.15.98:0/2193128233 shutdown_connections 2024-08-14T06:52:25.490 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.488+0000 7fe77e30d640 1 --2- 172.21.15.98:0/2193128233 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fe77815c8a0 0x7fe77806e700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:25.490 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.488+0000 7fe77e30d640 1 --2- 172.21.15.98:0/2193128233 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fe778072890 0x7fe778074620 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:25.490 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.488+0000 7fe77e30d640 1 -- 172.21.15.98:0/2193128233 >> 172.21.15.98:0/2193128233 conn(0x7fe7780cfea0 msgr2=0x7fe7780d02b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:25.490 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.488+0000 7fe77e30d640 1 -- 172.21.15.98:0/2193128233 shutdown_connections 2024-08-14T06:52:25.490 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.488+0000 7fe77e30d640 1 -- 172.21.15.98:0/2193128233 wait complete. 2024-08-14T06:52:25.491 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.490+0000 7fe77e30d640 1 Processor -- start 2024-08-14T06:52:25.491 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.490+0000 7fe77e30d640 1 -- start start 2024-08-14T06:52:25.494 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe77e30d640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fe77806ec40 0x7fe7781515e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:25.494 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe77e30d640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fe778072890 0x7fe7781f5ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:25.494 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe77e30d640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fe77815c8a0 0x7fe7781f9990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:25.494 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe77e30d640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7fe7780ae840 con 0x7fe77806ec40 2024-08-14T06:52:25.495 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe77e30d640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7fe7780ae6c0 con 0x7fe778072890 2024-08-14T06:52:25.495 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe77e30d640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7fe7780ae9c0 con 0x7fe77815c8a0 2024-08-14T06:52:25.495 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe777fff640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fe77806ec40 0x7fe7781515e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:25.495 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe777fff640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fe77806ec40 0x7fe7781515e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:44776/0 (socket says 172.21.15.98:44776) 2024-08-14T06:52:25.495 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe777fff640 1 -- 172.21.15.98:0/3912181884 learned_addr learned my addr 172.21.15.98:0/3912181884 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:25.495 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe77c8aa640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fe77815c8a0 0x7fe7781f9990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:25.495 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.491+0000 7fe7777fe640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fe778072890 0x7fe7781f5ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:25.495 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.492+0000 7fe777fff640 1 -- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fe77815c8a0 msgr2=0x7fe7781f9990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:25.495 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.492+0000 7fe777fff640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fe77815c8a0 0x7fe7781f9990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:25.495 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.492+0000 7fe777fff640 1 -- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fe778072890 msgr2=0x7fe7781f5ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:25.496 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.492+0000 7fe777fff640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fe778072890 0x7fe7781f5ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:25.496 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.492+0000 7fe777fff640 1 -- 172.21.15.98:0/3912181884 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe778152880 con 0x7fe77806ec40 2024-08-14T06:52:25.496 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.492+0000 7fe77c8aa640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fe77815c8a0 0x7fe7781f9990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-08-14T06:52:25.496 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.492+0000 7fe777fff640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fe77806ec40 0x7fe7781515e0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fe768006010 tx=0x7fe7680038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:25.537 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.535+0000 7fe774ff9640 1 -- 172.21.15.98:0/3912181884 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe76803e030 con 0x7fe77806ec40 2024-08-14T06:52:25.537 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.535+0000 7fe774ff9640 1 -- 172.21.15.98:0/3912181884 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe768003d90 con 0x7fe77806ec40 2024-08-14T06:52:25.537 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.535+0000 7fe774ff9640 1 -- 172.21.15.98:0/3912181884 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe768035040 con 0x7fe77806ec40 2024-08-14T06:52:25.537 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.535+0000 7fe77e30d640 1 -- 172.21.15.98:0/3912181884 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe778152b40 con 0x7fe77806ec40 2024-08-14T06:52:25.537 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.535+0000 7fe77e30d640 1 -- 172.21.15.98:0/3912181884 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe7781f9470 con 0x7fe77806ec40 2024-08-14T06:52:25.538 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.536+0000 7fe77e30d640 1 -- 172.21.15.98:0/3912181884 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe778074620 con 0x7fe77806ec40 2024-08-14T06:52:25.539 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.537+0000 7fe774ff9640 1 -- 172.21.15.98:0/3912181884 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 4) ==== 99083+0+0 (secure 0 0 0) 0x7fe768041030 con 0x7fe77806ec40 2024-08-14T06:52:25.539 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.538+0000 7fe774ff9640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fe764076f80 0x7fe764079490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:25.539 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.538+0000 7fe774ff9640 1 -- 172.21.15.98:0/3912181884 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(13..13 src has 1..13) ==== 2511+0+0 (secure 0 0 0) 0x7fe7680b8160 con 0x7fe77806ec40 2024-08-14T06:52:25.539 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.538+0000 7fe7777fe640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fe764076f80 0x7fe764079490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:25.540 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.538+0000 7fe7777fe640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fe764076f80 0x7fe764079490 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fe7781f0210 tx=0x7fe76c005f30 comp rx=0 tx=0).ready entity=mgr.4109 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:25.544 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.543+0000 7fe774ff9640 1 -- 172.21.15.98:0/3912181884 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+194988 (secure 0 0 0) 0x7fe77806f050 con 0x7fe77806ec40 2024-08-14T06:52:25.761 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.759+0000 7fe77e30d640 1 -- 172.21.15.98:0/3912181884 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fe7780b5e20 con 0x7fe77806ec40 2024-08-14T06:52:25.761 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.760+0000 7fe774ff9640 1 -- 172.21.15.98:0/3912181884 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v13) ==== 74+0+7567 (secure 0 0 0) 0x7fe76808a390 con 0x7fe77806ec40 2024-08-14T06:52:25.762 INFO:teuthology.misc.health.smithi098.stdout: 2024-08-14T06:52:25.762 INFO:teuthology.misc.health.smithi098.stdout:{"epoch":13,"fsid":"1a641e76-d282-43fb-b02c-28f3852c1ab1","created":"2024-08-14T06:52:14.033527+0000","modified":"2024-08-14T06:52:24.453181+0000","last_up_change":"2024-08-14T06:52:24.453181+0000","last_in_change":"2024-08-14T06:52:17.300583+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"273f37e4-1d4c-46cd-96bc-3d7fce969572","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"ebd9bc08-f436-4e26-a0ba-d79a2220ba5e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"9979a52a-0573-4019-a471-2ad384d88176","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"aa26d769-fd1e-4963-af87-a7006f935553","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"0cd7a5fd-20b0-40f2-9e45-fd71ceadb6e3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"fbc7b048-ee2c-46e1-81e9-854d0f132434","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"3f5d0d02-eae3-4d2b-b611-03cbec6590a4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"8d15135a-d5ad-47b1-988a-c01ccd36426e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6801","nonce":2707254681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6803","nonce":2707254681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6807","nonce":2707254681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6805","nonce":2707254681}]},"public_addr":"172.21.15.191:6801/2707254681","cluster_addr":"172.21.15.191:6803/2707254681","heartbeat_back_addr":"172.21.15.191:6807/2707254681","heartbeat_front_addr":"172.21.15.191:6805/2707254681","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-14T06:52:25.765 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.763+0000 7fe77e30d640 1 -- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fe764076f80 msgr2=0x7fe764079490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:25.765 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.763+0000 7fe77e30d640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fe764076f80 0x7fe764079490 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fe7781f0210 tx=0x7fe76c005f30 comp rx=0 tx=0).stop 2024-08-14T06:52:25.765 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.763+0000 7fe77e30d640 1 -- 172.21.15.98:0/3912181884 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fe77806ec40 msgr2=0x7fe7781515e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:25.765 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.763+0000 7fe77e30d640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fe77806ec40 0x7fe7781515e0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fe768006010 tx=0x7fe7680038a0 comp rx=0 tx=0).stop 2024-08-14T06:52:25.765 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.763+0000 7fe77e30d640 1 -- 172.21.15.98:0/3912181884 shutdown_connections 2024-08-14T06:52:25.765 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.763+0000 7fe77e30d640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fe764076f80 0x7fe764079490 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:25.765 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.763+0000 7fe77e30d640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fe77815c8a0 0x7fe7781f9990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:25.765 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.763+0000 7fe77e30d640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fe778072890 0x7fe7781f5ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:25.765 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.763+0000 7fe77e30d640 1 --2- 172.21.15.98:0/3912181884 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fe77806ec40 0x7fe7781515e0 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:25.766 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.763+0000 7fe77e30d640 1 -- 172.21.15.98:0/3912181884 >> 172.21.15.98:0/3912181884 conn(0x7fe7780cfea0 msgr2=0x7fe7780cade0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:25.768 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.766+0000 7fe77e30d640 1 -- 172.21.15.98:0/3912181884 shutdown_connections 2024-08-14T06:52:25.768 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:25.766+0000 7fe77e30d640 1 -- 172.21.15.98:0/3912181884 wait complete. 2024-08-14T06:52:25.781 DEBUG:teuthology.misc:1 of 8 OSDs are up 2024-08-14T06:52:26.140 INFO:tasks.ceph.osd.5.smithi191.stderr:2024-08-14T06:52:26.139+0000 7f012d650640 -1 osd.5 0 waiting for initial osdmap 2024-08-14T06:52:26.151 INFO:tasks.ceph.osd.4.smithi191.stderr:2024-08-14T06:52:26.150+0000 7fbc17237640 -1 osd.4 0 waiting for initial osdmap 2024-08-14T06:52:26.155 INFO:tasks.ceph.osd.5.smithi191.stderr:2024-08-14T06:52:26.155+0000 7f0125446640 -1 osd.5 14 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-14T06:52:26.160 INFO:tasks.ceph.osd.4.smithi191.stderr:2024-08-14T06:52:26.160+0000 7fbc0f02d640 -1 osd.4 14 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-14T06:52:26.213 INFO:tasks.ceph.osd.6.smithi191.stderr:2024-08-14T06:52:26.213+0000 7f7d41b23640 -1 osd.6 0 waiting for initial osdmap 2024-08-14T06:52:26.227 INFO:tasks.ceph.osd.6.smithi191.stderr:2024-08-14T06:52:26.227+0000 7f7d39919640 -1 osd.6 14 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-14T06:52:26.454 INFO:tasks.ceph.osd.0.smithi098.stderr:2024-08-14T06:52:26.452+0000 7f6ce9b1d640 -1 osd.0 0 waiting for initial osdmap 2024-08-14T06:52:26.469 INFO:tasks.ceph.osd.0.smithi098.stderr:2024-08-14T06:52:26.468+0000 7f6ce1913640 -1 osd.0 14 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-14T06:52:26.558 INFO:tasks.ceph.osd.1.smithi098.stderr:2024-08-14T06:52:26.556+0000 7ffa8a806640 -1 osd.1 0 waiting for initial osdmap 2024-08-14T06:52:26.560 INFO:tasks.ceph.osd.2.smithi098.stderr:2024-08-14T06:52:26.558+0000 7f83ccc8b640 -1 osd.2 0 waiting for initial osdmap 2024-08-14T06:52:26.574 INFO:tasks.ceph.osd.1.smithi098.stderr:2024-08-14T06:52:26.573+0000 7ffa825fc640 -1 osd.1 14 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-14T06:52:26.584 INFO:tasks.ceph.osd.2.smithi098.stderr:2024-08-14T06:52:26.583+0000 7f83c4a81640 -1 osd.2 14 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-14T06:52:26.633 INFO:tasks.ceph.osd.3.smithi098.stderr:2024-08-14T06:52:26.631+0000 7f7130a94640 -1 osd.3 0 waiting for initial osdmap 2024-08-14T06:52:26.644 INFO:tasks.ceph.osd.3.smithi098.stderr:2024-08-14T06:52:26.643+0000 7f712888a640 -1 osd.3 14 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-14T06:52:32.588 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-08-14T06:52:32.699 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.698+0000 7fc21f550640 1 -- 172.21.15.98:0/899259940 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc218152250 msgr2=0x7fc21815bb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:32.700 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.698+0000 7fc21f550640 1 --2- 172.21.15.98:0/899259940 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc218152250 0x7fc21815bb00 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fc210005fd0 tx=0x7fc21002a980 comp rx=0 tx=0).stop 2024-08-14T06:52:32.700 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.698+0000 7fc21f550640 1 -- 172.21.15.98:0/899259940 shutdown_connections 2024-08-14T06:52:32.700 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.698+0000 7fc21f550640 1 --2- 172.21.15.98:0/899259940 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc218152250 0x7fc21815bb00 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:32.700 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.698+0000 7fc21f550640 1 --2- 172.21.15.98:0/899259940 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc2180cc190 0x7fc218073790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:32.700 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.698+0000 7fc21f550640 1 -- 172.21.15.98:0/899259940 >> 172.21.15.98:0/899259940 conn(0x7fc2180b2b30 msgr2=0x7fc2180b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:32.700 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.698+0000 7fc21f550640 1 -- 172.21.15.98:0/899259940 shutdown_connections 2024-08-14T06:52:32.700 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.698+0000 7fc21f550640 1 -- 172.21.15.98:0/899259940 wait complete. 2024-08-14T06:52:32.701 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.700+0000 7fc21f550640 1 Processor -- start 2024-08-14T06:52:32.701 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.700+0000 7fc21f550640 1 -- start start 2024-08-14T06:52:32.703 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.701+0000 7fc21f550640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc218073cd0 0x7fc2181f23a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:32.703 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.701+0000 7fc21f550640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc2180cc190 0x7fc218155130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:32.703 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.701+0000 7fc21f550640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc218152250 0x7fc2180c6f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:32.703 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.701+0000 7fc21f550640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7fc21815c450 con 0x7fc218152250 2024-08-14T06:52:32.703 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.701+0000 7fc21f550640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7fc21815c2d0 con 0x7fc2180cc190 2024-08-14T06:52:32.703 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.701+0000 7fc21f550640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7fc21815c5d0 con 0x7fc218073cd0 2024-08-14T06:52:32.703 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21daed640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc218152250 0x7fc2180c6f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21daed640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc218152250 0x7fc2180c6f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:44806/0 (socket says 172.21.15.98:44806) 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21daed640 1 -- 172.21.15.98:0/1897728243 learned_addr learned my addr 172.21.15.98:0/1897728243 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21daed640 1 -- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc218073cd0 msgr2=0x7fc2181f23a0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21daed640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc218073cd0 0x7fc2181f23a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21daed640 1 -- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc2180cc190 msgr2=0x7fc218155130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21daed640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc2180cc190 0x7fc218155130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21daed640 1 -- 172.21.15.98:0/1897728243 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc2181f2d10 con 0x7fc218152250 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21daed640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc218152250 0x7fc2180c6f50 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fc210033040 tx=0x7fc210003f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc20dffb640 1 -- 172.21.15.98:0/1897728243 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc210034040 con 0x7fc218152250 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc20dffb640 1 -- 172.21.15.98:0/1897728243 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc210004640 con 0x7fc218152250 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc20dffb640 1 -- 172.21.15.98:0/1897728243 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc210045480 con 0x7fc218152250 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21f550640 1 -- 172.21.15.98:0/1897728243 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc2181f2ff0 con 0x7fc218152250 2024-08-14T06:52:32.704 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.702+0000 7fc21f550640 1 -- 172.21.15.98:0/1897728243 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc218155670 con 0x7fc218152250 2024-08-14T06:52:32.705 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.703+0000 7fc21f550640 1 -- 172.21.15.98:0/1897728243 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc2181f2a80 con 0x7fc218152250 2024-08-14T06:52:32.705 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.704+0000 7fc20dffb640 1 -- 172.21.15.98:0/1897728243 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 5) ==== 99139+0+0 (secure 0 0 0) 0x7fc210037020 con 0x7fc218152250 2024-08-14T06:52:32.705 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.704+0000 7fc20dffb640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc1fc077010 0x7fc1fc079520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:32.705 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.704+0000 7fc20dffb640 1 -- 172.21.15.98:0/1897728243 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 4857+0+0 (secure 0 0 0) 0x7fc2100b9e60 con 0x7fc218152250 2024-08-14T06:52:32.706 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.705+0000 7fc21d2ec640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc1fc077010 0x7fc1fc079520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:32.706 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.706+0000 7fc21d2ec640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc1fc077010 0x7fc1fc079520 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fc214009610 tx=0x7fc214006b80 comp rx=0 tx=0).ready entity=mgr.4109 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:32.710 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.709+0000 7fc20dffb640 1 -- 172.21.15.98:0/1897728243 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+194988 (secure 0 0 0) 0x7fc2100834d0 con 0x7fc218152250 2024-08-14T06:52:32.895 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.894+0000 7fc21f550640 1 -- 172.21.15.98:0/1897728243 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fc218073790 con 0x7fc218152250 2024-08-14T06:52:32.896 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.895+0000 7fc20dffb640 1 -- 172.21.15.98:0/1897728243 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v19) ==== 74+0+12920 (secure 0 0 0) 0x7fc210082dd0 con 0x7fc218152250 2024-08-14T06:52:32.896 INFO:teuthology.misc.health.smithi098.stdout: 2024-08-14T06:52:32.896 INFO:teuthology.misc.health.smithi098.stdout:{"epoch":19,"fsid":"1a641e76-d282-43fb-b02c-28f3852c1ab1","created":"2024-08-14T06:52:14.033527+0000","modified":"2024-08-14T06:52:31.172040+0000","last_up_change":"2024-08-14T06:52:27.155904+0000","last_in_change":"2024-08-14T06:52:17.300583+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-14T06:52:28.459302+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"273f37e4-1d4c-46cd-96bc-3d7fce969572","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6809","nonce":3122853447}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6811","nonce":3122853447}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6815","nonce":3122853447}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6813","nonce":3122853447}]},"public_addr":"172.21.15.98:6809/3122853447","cluster_addr":"172.21.15.98:6811/3122853447","heartbeat_back_addr":"172.21.15.98:6815/3122853447","heartbeat_front_addr":"172.21.15.98:6813/3122853447","state":["exists","up"]},{"osd":1,"uuid":"ebd9bc08-f436-4e26-a0ba-d79a2220ba5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":17,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6817","nonce":4048855513}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6819","nonce":4048855513}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6823","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6825","nonce":4048855513}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6821","nonce":4048855513}]},"public_addr":"172.21.15.98:6817/4048855513","cluster_addr":"172.21.15.98:6819/4048855513","heartbeat_back_addr":"172.21.15.98:6825/4048855513","heartbeat_front_addr":"172.21.15.98:6821/4048855513","state":["exists","up"]},{"osd":2,"uuid":"9979a52a-0573-4019-a471-2ad384d88176","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6824","nonce":2997064267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6827","nonce":2997064267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6831","nonce":2997064267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6829","nonce":2997064267}]},"public_addr":"172.21.15.98:6824/2997064267","cluster_addr":"172.21.15.98:6827/2997064267","heartbeat_back_addr":"172.21.15.98:6831/2997064267","heartbeat_front_addr":"172.21.15.98:6829/2997064267","state":["exists","up"]},{"osd":3,"uuid":"aa26d769-fd1e-4963-af87-a7006f935553","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6800","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6801","nonce":3163598583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6803","nonce":3163598583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6807","nonce":3163598583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6805","nonce":3163598583}]},"public_addr":"172.21.15.98:6801/3163598583","cluster_addr":"172.21.15.98:6803/3163598583","heartbeat_back_addr":"172.21.15.98:6807/3163598583","heartbeat_front_addr":"172.21.15.98:6805/3163598583","state":["exists","up"]},{"osd":4,"uuid":"0cd7a5fd-20b0-40f2-9e45-fd71ceadb6e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6819","nonce":3843105526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6821","nonce":3843105526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6825","nonce":3843105526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6823","nonce":3843105526}]},"public_addr":"172.21.15.191:6819/3843105526","cluster_addr":"172.21.15.191:6821/3843105526","heartbeat_back_addr":"172.21.15.191:6825/3843105526","heartbeat_front_addr":"172.21.15.191:6823/3843105526","state":["exists","up"]},{"osd":5,"uuid":"fbc7b048-ee2c-46e1-81e9-854d0f132434","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6827","nonce":2246847695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6829","nonce":2246847695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6832","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6833","nonce":2246847695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6831","nonce":2246847695}]},"public_addr":"172.21.15.191:6827/2246847695","cluster_addr":"172.21.15.191:6829/2246847695","heartbeat_back_addr":"172.21.15.191:6833/2246847695","heartbeat_front_addr":"172.21.15.191:6831/2246847695","state":["exists","up"]},{"osd":6,"uuid":"3f5d0d02-eae3-4d2b-b611-03cbec6590a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6811","nonce":2491745875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6813","nonce":2491745875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6817","nonce":2491745875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6815","nonce":2491745875}]},"public_addr":"172.21.15.191:6811/2491745875","cluster_addr":"172.21.15.191:6813/2491745875","heartbeat_back_addr":"172.21.15.191:6817/2491745875","heartbeat_front_addr":"172.21.15.191:6815/2491745875","state":["exists","up"]},{"osd":7,"uuid":"8d15135a-d5ad-47b1-988a-c01ccd36426e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6801","nonce":2707254681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6803","nonce":2707254681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6807","nonce":2707254681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6805","nonce":2707254681}]},"public_addr":"172.21.15.191:6801/2707254681","cluster_addr":"172.21.15.191:6803/2707254681","heartbeat_back_addr":"172.21.15.191:6807/2707254681","heartbeat_front_addr":"172.21.15.191:6805/2707254681","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.934566+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.154949+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.113670+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.445209+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.574716+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.080765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.505776+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.103787+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-14T06:52:32.898 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 -- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc1fc077010 msgr2=0x7fc1fc079520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc1fc077010 0x7fc1fc079520 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fc214009610 tx=0x7fc214006b80 comp rx=0 tx=0).stop 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 -- 172.21.15.98:0/1897728243 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc218152250 msgr2=0x7fc2180c6f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc218152250 0x7fc2180c6f50 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fc210033040 tx=0x7fc210003f10 comp rx=0 tx=0).stop 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 -- 172.21.15.98:0/1897728243 shutdown_connections 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc1fc077010 0x7fc1fc079520 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc218152250 0x7fc2180c6f50 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc2180cc190 0x7fc218155130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 --2- 172.21.15.98:0/1897728243 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc218073cd0 0x7fc2181f23a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 -- 172.21.15.98:0/1897728243 >> 172.21.15.98:0/1897728243 conn(0x7fc2180b2b30 msgr2=0x7fc2180cef30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.897+0000 7fc21f550640 1 -- 172.21.15.98:0/1897728243 shutdown_connections 2024-08-14T06:52:32.899 INFO:teuthology.misc.health.smithi098.stderr:2024-08-14T06:52:32.898+0000 7fc21f550640 1 -- 172.21.15.98:0/1897728243 wait complete. 2024-08-14T06:52:32.908 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-08-14T06:52:32.909 INFO:tasks.ceph:Starting mds daemons in cluster ceph... 2024-08-14T06:52:32.909 INFO:tasks.ceph.mds.a:Restarting daemon 2024-08-14T06:52:32.909 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-08-14T06:52:32.911 INFO:tasks.ceph.mds.a:Started 2024-08-14T06:52:32.911 INFO:tasks.ceph.mds.c:Restarting daemon 2024-08-14T06:52:32.912 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-08-14T06:52:32.914 INFO:tasks.ceph.mds.c:Started 2024-08-14T06:52:32.914 INFO:tasks.ceph.mds.b:Restarting daemon 2024-08-14T06:52:32.914 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-08-14T06:52:32.917 INFO:tasks.ceph.mds.b:Started 2024-08-14T06:52:32.917 INFO:tasks.ceph.mds.d:Restarting daemon 2024-08-14T06:52:32.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-08-14T06:52:32.920 INFO:tasks.ceph.mds.d:Started 2024-08-14T06:52:32.920 INFO:tasks.ceph:Setting up CephFS filesystem(s)... 2024-08-14T06:52:32.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-08-14T06:52:32.998 INFO:tasks.ceph.mds.a.smithi098.stdout:starting mds.a at 2024-08-14T06:52:32.998 INFO:tasks.ceph.mds.c.smithi098.stdout:starting mds.c at 2024-08-14T06:52:33.018 INFO:tasks.ceph.mds.d.smithi191.stdout:starting mds.d at 2024-08-14T06:52:33.018 INFO:tasks.ceph.mds.b.smithi191.stdout:starting mds.b at 2024-08-14T06:52:33.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.040+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/1648383454 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9acc0cc190 msgr2=0x7f9acc073790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:33.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.040+0000 7f9ad2ad7640 1 --2- 172.21.15.98:0/1648383454 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9acc0cc190 0x7f9acc073790 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f9ac0009830 tx=0x7f9ac002a4a0 comp rx=0 tx=0).stop 2024-08-14T06:52:33.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.040+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/1648383454 shutdown_connections 2024-08-14T06:52:33.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.040+0000 7f9ad2ad7640 1 --2- 172.21.15.98:0/1648383454 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9acc0cc190 0x7f9acc073790 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:33.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.040+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/1648383454 >> 172.21.15.98:0/1648383454 conn(0x7f9acc0b2b30 msgr2=0x7f9acc0b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:33.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.040+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/1648383454 shutdown_connections 2024-08-14T06:52:33.042 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.040+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/1648383454 wait complete. 2024-08-14T06:52:33.043 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.042+0000 7f9ad2ad7640 1 Processor -- start 2024-08-14T06:52:33.043 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.042+0000 7f9ad2ad7640 1 -- start start 2024-08-14T06:52:33.046 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.043+0000 7f9ad2ad7640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9acc073cd0 0x7f9acc150630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:33.046 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.043+0000 7f9ad2ad7640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9acc0cc190 0x7f9acc158220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:33.046 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.043+0000 7f9ad2ad7640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9acc152250 0x7f9acc158760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:33.046 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.043+0000 7f9ad2ad7640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f9acc15c210 con 0x7f9acc073cd0 2024-08-14T06:52:33.046 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.043+0000 7f9ad2ad7640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f9acc0d00e0 con 0x7f9acc0cc190 2024-08-14T06:52:33.047 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.043+0000 7f9ad2ad7640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f9acc15c390 con 0x7f9acc152250 2024-08-14T06:52:33.047 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.043+0000 7f9ad0873640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9acc073cd0 0x7f9acc150630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:33.047 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad0873640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9acc073cd0 0x7f9acc150630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:44858/0 (socket says 172.21.15.98:44858) 2024-08-14T06:52:33.047 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad0873640 1 -- 172.21.15.98:0/2801561033 learned_addr learned my addr 172.21.15.98:0/2801561033 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:33.047 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad1074640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9acc152250 0x7f9acc158760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:33.047 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9acbfff640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9acc0cc190 0x7f9acc158220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:33.047 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad0873640 1 -- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9acc152250 msgr2=0x7f9acc158760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:33.047 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad0873640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9acc152250 0x7f9acc158760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:33.047 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad0873640 1 -- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9acc0cc190 msgr2=0x7f9acc158220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:33.047 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad0873640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9acc0cc190 0x7f9acc158220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:33.048 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad0873640 1 -- 172.21.15.98:0/2801561033 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9acc0c5100 con 0x7f9acc073cd0 2024-08-14T06:52:33.048 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad1074640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9acc152250 0x7f9acc158760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-08-14T06:52:33.048 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9acbfff640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9acc0cc190 0x7f9acc158220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-08-14T06:52:33.048 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad0873640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9acc073cd0 0x7f9acc150630 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f9ac002aa90 tx=0x7f9ac0033c20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:33.048 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ac97fa640 1 -- 172.21.15.98:0/2801561033 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9ac0005c90 con 0x7f9acc073cd0 2024-08-14T06:52:33.048 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ac97fa640 1 -- 172.21.15.98:0/2801561033 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9ac0031e50 con 0x7f9acc073cd0 2024-08-14T06:52:33.048 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/2801561033 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9acc150b70 con 0x7f9acc073cd0 2024-08-14T06:52:33.048 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ac97fa640 1 -- 172.21.15.98:0/2801561033 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9ac0005260 con 0x7f9acc073cd0 2024-08-14T06:52:33.048 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.044+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/2801561033 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9acc151360 con 0x7f9acc073cd0 2024-08-14T06:52:33.048 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.045+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/2801561033 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9acc0c52d0 con 0x7f9acc073cd0 2024-08-14T06:52:33.049 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.046+0000 7f9ac97fa640 1 -- 172.21.15.98:0/2801561033 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 5) ==== 99139+0+0 (secure 0 0 0) 0x7f9ac0005400 con 0x7f9acc073cd0 2024-08-14T06:52:33.049 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.047+0000 7f9ac97fa640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f9ab0077010 0x7f9ab0079520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:33.049 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.047+0000 7f9ac97fa640 1 -- 172.21.15.98:0/2801561033 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 4857+0+0 (secure 0 0 0) 0x7f9ac00b95a0 con 0x7f9acc073cd0 2024-08-14T06:52:33.049 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.047+0000 7f9acbfff640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f9ab0077010 0x7f9ab0079520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:33.050 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.047+0000 7f9acbfff640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f9ab0077010 0x7f9ab0079520 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f9ab8001120 tx=0x7f9ab8009040 comp rx=0 tx=0).ready entity=mgr.4109 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:33.052 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.051+0000 7f9ac97fa640 1 -- 172.21.15.98:0/2801561033 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+194988 (secure 0 0 0) 0x7f9ac003f030 con 0x7f9acc073cd0 2024-08-14T06:52:33.275 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.274+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/2801561033 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f9acc073790 con 0x7f9acc073cd0 2024-08-14T06:52:33.275 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.275+0000 7f9ac97fa640 1 -- 172.21.15.98:0/2801561033 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 2 v2) ==== 93+0+1434 (secure 0 0 0) 0x7f9ac0082b80 con 0x7f9acc073cd0 2024-08-14T06:52:33.276 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:52:33.276 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":2,"btime":"2024-08-14T06:52:33:003067+0000","default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4353,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.98:6834/1565392435","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":1565392435},{"type":"v1","addr":"172.21.15.98:6834","nonce":1565392435}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[]} 2024-08-14T06:52:33.276 INFO:teuthology.orchestra.run.smithi098.stderr:dumped fsmap epoch 2 2024-08-14T06:52:33.278 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f9ab0077010 msgr2=0x7f9ab0079520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:33.278 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f9ab0077010 0x7f9ab0079520 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f9ab8001120 tx=0x7f9ab8009040 comp rx=0 tx=0).stop 2024-08-14T06:52:33.278 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/2801561033 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9acc073cd0 msgr2=0x7f9acc150630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:33.278 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9acc073cd0 0x7f9acc150630 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f9ac002aa90 tx=0x7f9ac0033c20 comp rx=0 tx=0).stop 2024-08-14T06:52:33.278 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/2801561033 shutdown_connections 2024-08-14T06:52:33.278 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f9ab0077010 0x7f9ab0079520 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:33.278 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f9acc152250 0x7f9acc158760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:33.278 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f9acc0cc190 0x7f9acc158220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:33.279 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 --2- 172.21.15.98:0/2801561033 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f9acc073cd0 0x7f9acc150630 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:33.279 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/2801561033 >> 172.21.15.98:0/2801561033 conn(0x7f9acc0b2b30 msgr2=0x7f9acc0afe20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:33.279 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/2801561033 shutdown_connections 2024-08-14T06:52:33.279 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:33.277+0000 7f9ad2ad7640 1 -- 172.21.15.98:0/2801561033 wait complete. 2024-08-14T06:52:33.855 DEBUG:teuthology.orchestra.run.smithi098:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-08-14T06:52:33.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-08-14T06:52:35.289 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-08-14T06:52:35.417 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.415+0000 7f385f79d640 1 -- 172.21.15.98:0/2748003029 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3858152080 msgr2=0x7f385815bb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:35.417 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.415+0000 7f385f79d640 1 --2- 172.21.15.98:0/2748003029 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3858152080 0x7f385815bb00 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f3858073340 tx=0x7f385002a8f0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.417 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.415+0000 7f385f79d640 1 -- 172.21.15.98:0/2748003029 shutdown_connections 2024-08-14T06:52:35.417 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.415+0000 7f385f79d640 1 --2- 172.21.15.98:0/2748003029 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3858152080 0x7f385815bb00 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.417 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.415+0000 7f385f79d640 1 --2- 172.21.15.98:0/2748003029 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3858073cd0 0x7f3858151b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.417 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.415+0000 7f385f79d640 1 --2- 172.21.15.98:0/2748003029 >> v2:172.21.15.191:3300/0 conn(0x7f38580cc190 0x7f3858073790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.417 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.415+0000 7f385f79d640 1 -- 172.21.15.98:0/2748003029 >> 172.21.15.98:0/2748003029 conn(0x7f38580b2b30 msgr2=0x7f38580b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:35.417 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.416+0000 7f385f79d640 1 -- 172.21.15.98:0/2748003029 shutdown_connections 2024-08-14T06:52:35.417 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.416+0000 7f385f79d640 1 -- 172.21.15.98:0/2748003029 wait complete. 2024-08-14T06:52:35.418 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.417+0000 7f385f79d640 1 Processor -- start 2024-08-14T06:52:35.418 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.417+0000 7f385f79d640 1 -- start start 2024-08-14T06:52:35.421 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385f79d640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3858073cd0 0x7f3858152940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:35.421 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385f79d640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f38580cc190 0x7f38580b6b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:35.421 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385f79d640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3858152080 0x7f3858150f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:35.421 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385f79d640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f385815c1c0 con 0x7f3858152080 2024-08-14T06:52:35.421 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385f79d640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f385815c040 con 0x7f38580cc190 2024-08-14T06:52:35.421 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385f79d640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f385815c340 con 0x7f3858073cd0 2024-08-14T06:52:35.421 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385dd3a640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3858152080 0x7f3858150f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:35.422 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385dd3a640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3858152080 0x7f3858150f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:41998/0 (socket says 172.21.15.98:41998) 2024-08-14T06:52:35.422 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385dd3a640 1 -- 172.21.15.98:0/3836248483 learned_addr learned my addr 172.21.15.98:0/3836248483 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:35.422 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385cd38640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f38580cc190 0x7f38580b6b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:35.422 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385d539640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3858073cd0 0x7f3858152940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:35.422 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385dd3a640 1 -- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3858073cd0 msgr2=0x7f3858152940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:35.422 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385dd3a640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3858073cd0 0x7f3858152940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.422 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385dd3a640 1 -- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f38580cc190 msgr2=0x7f38580b6b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:35.422 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385dd3a640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f38580cc190 0x7f38580b6b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.422 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385dd3a640 1 -- 172.21.15.98:0/3836248483 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3858156df0 con 0x7f3858152080 2024-08-14T06:52:35.422 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385dd3a640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3858152080 0x7f3858150f40 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f3850033040 tx=0x7f3850003c20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:35.423 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f384dffb640 1 -- 172.21.15.98:0/3836248483 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3850004240 con 0x7f3858152080 2024-08-14T06:52:35.423 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f384dffb640 1 -- 172.21.15.98:0/3836248483 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f385002d070 con 0x7f3858152080 2024-08-14T06:52:35.423 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f385f79d640 1 -- 172.21.15.98:0/3836248483 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3858157a00 con 0x7f3858152080 2024-08-14T06:52:35.423 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.419+0000 7f384dffb640 1 -- 172.21.15.98:0/3836248483 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f38500046a0 con 0x7f3858152080 2024-08-14T06:52:35.423 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.420+0000 7f385f79d640 1 -- 172.21.15.98:0/3836248483 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3858157fc0 con 0x7f3858152080 2024-08-14T06:52:35.424 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.421+0000 7f385f79d640 1 -- 172.21.15.98:0/3836248483 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3858156b60 con 0x7f3858152080 2024-08-14T06:52:35.424 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.421+0000 7f384dffb640 1 -- 172.21.15.98:0/3836248483 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 5) ==== 99139+0+0 (secure 0 0 0) 0x7f3850031480 con 0x7f3858152080 2024-08-14T06:52:35.424 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.422+0000 7f384dffb640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f383c077010 0x7f383c079520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:35.424 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.422+0000 7f384dffb640 1 -- 172.21.15.98:0/3836248483 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 4857+0+0 (secure 0 0 0) 0x7f38500b9e80 con 0x7f3858152080 2024-08-14T06:52:35.424 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.422+0000 7f385d539640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f383c077010 0x7f383c079520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:35.424 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.423+0000 7f385d539640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f383c077010 0x7f383c079520 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f385400bad0 tx=0x7f385400b2e0 comp rx=0 tx=0).ready entity=mgr.4109 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:35.427 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.426+0000 7f384dffb640 1 -- 172.21.15.98:0/3836248483 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+194988 (secure 0 0 0) 0x7f3850082d80 con 0x7f3858152080 2024-08-14T06:52:35.650 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.650+0000 7f385f79d640 1 -- 172.21.15.98:0/3836248483 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f3858073790 con 0x7f3858152080 2024-08-14T06:52:35.651 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.650+0000 7f384dffb640 1 -- 172.21.15.98:0/3836248483 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 3 v3) ==== 93+0+3912 (secure 0 0 0) 0x7f38500872d0 con 0x7f3858152080 2024-08-14T06:52:35.651 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:52:35.651 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":3,"btime":"2024-08-14T06:52:34:004148+0000","default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4301,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.191:6837/3000311918","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6836","nonce":3000311918},{"type":"v1","addr":"172.21.15.191:6837","nonce":3000311918}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":4353,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.98:6834/1565392435","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":1565392435},{"type":"v1","addr":"172.21.15.98:6834","nonce":1565392435}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":4356,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.98:6835/1746444642","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6833","nonce":1746444642},{"type":"v1","addr":"172.21.15.98:6835","nonce":1746444642}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":4369,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.191:6835/4015991954","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6834","nonce":4015991954},{"type":"v1","addr":"172.21.15.191:6835","nonce":4015991954}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[]} 2024-08-14T06:52:35.652 INFO:teuthology.orchestra.run.smithi098.stderr:dumped fsmap epoch 3 2024-08-14T06:52:35.653 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.652+0000 7f385f79d640 1 -- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f383c077010 msgr2=0x7f383c079520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.652+0000 7f385f79d640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f383c077010 0x7f383c079520 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f385400bad0 tx=0x7f385400b2e0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.653+0000 7f385f79d640 1 -- 172.21.15.98:0/3836248483 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3858152080 msgr2=0x7f3858150f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.653+0000 7f385f79d640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3858152080 0x7f3858150f40 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f3850033040 tx=0x7f3850003c20 comp rx=0 tx=0).stop 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.653+0000 7f385f79d640 1 -- 172.21.15.98:0/3836248483 shutdown_connections 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.653+0000 7f385f79d640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f383c077010 0x7f383c079520 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.653+0000 7f385f79d640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f3858152080 0x7f3858150f40 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.653+0000 7f385f79d640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f38580cc190 0x7f38580b6b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.653+0000 7f385f79d640 1 --2- 172.21.15.98:0/3836248483 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f3858073cd0 0x7f3858152940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.653+0000 7f385f79d640 1 -- 172.21.15.98:0/3836248483 >> 172.21.15.98:0/3836248483 conn(0x7f38580b2b30 msgr2=0x7f38580afe20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.653+0000 7f385f79d640 1 -- 172.21.15.98:0/3836248483 shutdown_connections 2024-08-14T06:52:35.654 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.653+0000 7f385f79d640 1 -- 172.21.15.98:0/3836248483 wait complete. 2024-08-14T06:52:35.665 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-08-14T06:52:35.779 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.777+0000 7fc0801f3640 1 -- 172.21.15.98:0/3268998320 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc0780d18c0 msgr2=0x7fc0780d4110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:35.779 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.777+0000 7fc0801f3640 1 --2- 172.21.15.98:0/3268998320 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc0780d18c0 0x7fc0780d4110 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7fc070009830 tx=0x7fc07002a540 comp rx=0 tx=0).stop 2024-08-14T06:52:35.779 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.777+0000 7fc0801f3640 1 -- 172.21.15.98:0/3268998320 shutdown_connections 2024-08-14T06:52:35.779 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.777+0000 7fc0801f3640 1 --2- 172.21.15.98:0/3268998320 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc0780d18c0 0x7fc0780d4110 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.779 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.777+0000 7fc0801f3640 1 -- 172.21.15.98:0/3268998320 >> 172.21.15.98:0/3268998320 conn(0x7fc07808d9d0 msgr2=0x7fc07809d600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:35.780 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.777+0000 7fc0801f3640 1 -- 172.21.15.98:0/3268998320 shutdown_connections 2024-08-14T06:52:35.780 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.777+0000 7fc0801f3640 1 -- 172.21.15.98:0/3268998320 wait complete. 2024-08-14T06:52:35.784 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.779+0000 7fc0801f3640 1 Processor -- start 2024-08-14T06:52:35.784 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.779+0000 7fc0801f3640 1 -- start start 2024-08-14T06:52:35.785 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.780+0000 7fc0801f3640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc0780b54c0 0x7fc07815dc40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:35.785 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.780+0000 7fc0801f3640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc0780d18c0 0x7fc0781e7c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:35.785 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.780+0000 7fc0801f3640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc0780d4650 0x7fc0781e81d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:35.785 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.780+0000 7fc0801f3640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7fc0780b8ab0 con 0x7fc0780b54c0 2024-08-14T06:52:35.785 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.780+0000 7fc0801f3640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7fc0780b5ae0 con 0x7fc0780d18c0 2024-08-14T06:52:35.786 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.780+0000 7fc0801f3640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7fc0780b8c30 con 0x7fc0780d4650 2024-08-14T06:52:35.786 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.780+0000 7fc07df8f640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc0780b54c0 0x7fc07815dc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:35.786 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc07df8f640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc0780b54c0 0x7fc07815dc40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:42008/0 (socket says 172.21.15.98:42008) 2024-08-14T06:52:35.786 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc07df8f640 1 -- 172.21.15.98:0/606812557 learned_addr learned my addr 172.21.15.98:0/606812557 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:35.786 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc07d78e640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc0780d18c0 0x7fc0781e7c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:35.786 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc07e790640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc0780d4650 0x7fc0781e81d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:35.787 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc07df8f640 1 -- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc0780d4650 msgr2=0x7fc0781e81d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:35.787 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc07df8f640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc0780d4650 0x7fc0781e81d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.787 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc07df8f640 1 -- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc0780d18c0 msgr2=0x7fc0781e7c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:35.787 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc07df8f640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc0780d18c0 0x7fc0781e7c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.787 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc07df8f640 1 -- 172.21.15.98:0/606812557 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc0781ea9a0 con 0x7fc0780b54c0 2024-08-14T06:52:35.787 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc07df8f640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc0780b54c0 0x7fc07815dc40 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fc070009950 tx=0x7fc070003c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:35.788 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc0767fc640 1 -- 172.21.15.98:0/606812557 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc070005c90 con 0x7fc0780b54c0 2024-08-14T06:52:35.788 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc0767fc640 1 -- 172.21.15.98:0/606812557 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc070005020 con 0x7fc0780b54c0 2024-08-14T06:52:35.788 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc0801f3640 1 -- 172.21.15.98:0/606812557 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc0781eac80 con 0x7fc0780b54c0 2024-08-14T06:52:35.788 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc0767fc640 1 -- 172.21.15.98:0/606812557 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc070005450 con 0x7fc0780b54c0 2024-08-14T06:52:35.788 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.781+0000 7fc0801f3640 1 -- 172.21.15.98:0/606812557 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc0781eb240 con 0x7fc0780b54c0 2024-08-14T06:52:35.789 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.782+0000 7fc0801f3640 1 -- 172.21.15.98:0/606812557 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc07806a3c0 con 0x7fc0780b54c0 2024-08-14T06:52:35.789 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.783+0000 7fc0767fc640 1 -- 172.21.15.98:0/606812557 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 5) ==== 99139+0+0 (secure 0 0 0) 0x7fc0700317c0 con 0x7fc0780b54c0 2024-08-14T06:52:35.789 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.784+0000 7fc0767fc640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc064077010 0x7fc064079520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:35.789 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.784+0000 7fc0767fc640 1 -- 172.21.15.98:0/606812557 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 4857+0+0 (secure 0 0 0) 0x7fc0700b95d0 con 0x7fc0780b54c0 2024-08-14T06:52:35.789 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.784+0000 7fc07d78e640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc064077010 0x7fc064079520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:35.789 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.784+0000 7fc07d78e640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc064077010 0x7fc064079520 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fc06c007fd0 tx=0x7fc06c009040 comp rx=0 tx=0).ready entity=mgr.4109 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:35.790 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.788+0000 7fc0767fc640 1 -- 172.21.15.98:0/606812557 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+194988 (secure 0 0 0) 0x7fc07806a3c0 con 0x7fc0780b54c0 2024-08-14T06:52:35.975 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.974+0000 7fc0801f3640 1 -- 172.21.15.98:0/606812557 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) -- 0x7fc0780d4110 con 0x7fc0780b54c0 2024-08-14T06:52:35.975 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.974+0000 7fc0767fc640 1 -- 172.21.15.98:0/606812557 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v19) ==== 84+0+65 (secure 0 0 0) 0x7fc07003f030 con 0x7fc0780b54c0 2024-08-14T06:52:35.975 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:52:35.975 INFO:teuthology.orchestra.run.smithi098.stdout:[ 2024-08-14T06:52:35.976 INFO:teuthology.orchestra.run.smithi098.stdout: { 2024-08-14T06:52:35.976 INFO:teuthology.orchestra.run.smithi098.stdout: "poolnum": 1, 2024-08-14T06:52:35.976 INFO:teuthology.orchestra.run.smithi098.stdout: "poolname": ".mgr" 2024-08-14T06:52:35.976 INFO:teuthology.orchestra.run.smithi098.stdout: } 2024-08-14T06:52:35.976 INFO:teuthology.orchestra.run.smithi098.stdout:] 2024-08-14T06:52:35.977 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.976+0000 7fc0801f3640 1 -- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc064077010 msgr2=0x7fc064079520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.976+0000 7fc0801f3640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc064077010 0x7fc064079520 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fc06c007fd0 tx=0x7fc06c009040 comp rx=0 tx=0).stop 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.977+0000 7fc0801f3640 1 -- 172.21.15.98:0/606812557 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc0780b54c0 msgr2=0x7fc07815dc40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.977+0000 7fc0801f3640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc0780b54c0 0x7fc07815dc40 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fc070009950 tx=0x7fc070003c10 comp rx=0 tx=0).stop 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.977+0000 7fc0801f3640 1 -- 172.21.15.98:0/606812557 shutdown_connections 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.977+0000 7fc0801f3640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fc064077010 0x7fc064079520 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.977+0000 7fc0801f3640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fc0780d4650 0x7fc0781e81d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.977+0000 7fc0801f3640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fc0780d18c0 0x7fc0781e7c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.977+0000 7fc0801f3640 1 --2- 172.21.15.98:0/606812557 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fc0780b54c0 0x7fc07815dc40 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.977+0000 7fc0801f3640 1 -- 172.21.15.98:0/606812557 >> 172.21.15.98:0/606812557 conn(0x7fc07808d9d0 msgr2=0x7fc078161870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.977+0000 7fc0801f3640 1 -- 172.21.15.98:0/606812557 shutdown_connections 2024-08-14T06:52:35.978 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:35.977+0000 7fc0801f3640 1 -- 172.21.15.98:0/606812557 wait complete. 2024-08-14T06:52:35.988 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-08-14T06:52:35.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-08-14T06:52:36.106 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.105+0000 7f2e1621b640 1 -- 172.21.15.98:0/3575423620 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2e100cc190 msgr2=0x7f2e10073790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:36.106 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.105+0000 7f2e1621b640 1 --2- 172.21.15.98:0/3575423620 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2e100cc190 0x7f2e10073790 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f2e00009830 tx=0x7f2e0002a3f0 comp rx=0 tx=0).stop 2024-08-14T06:52:36.107 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.105+0000 7f2e1621b640 1 -- 172.21.15.98:0/3575423620 shutdown_connections 2024-08-14T06:52:36.107 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.105+0000 7f2e1621b640 1 --2- 172.21.15.98:0/3575423620 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f2e10152250 0x7f2e1015bb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:36.107 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.105+0000 7f2e1621b640 1 --2- 172.21.15.98:0/3575423620 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2e100cc190 0x7f2e10073790 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:36.107 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.105+0000 7f2e1621b640 1 -- 172.21.15.98:0/3575423620 >> 172.21.15.98:0/3575423620 conn(0x7f2e100b2b30 msgr2=0x7f2e100b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:36.107 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.105+0000 7f2e1621b640 1 -- 172.21.15.98:0/3575423620 shutdown_connections 2024-08-14T06:52:36.107 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.105+0000 7f2e1621b640 1 -- 172.21.15.98:0/3575423620 wait complete. 2024-08-14T06:52:36.107 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.107+0000 7f2e1621b640 1 Processor -- start 2024-08-14T06:52:36.108 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.107+0000 7f2e1621b640 1 -- start start 2024-08-14T06:52:36.110 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.108+0000 7f2e1621b640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f2e10073cd0 0x7f2e10150f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:36.110 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.108+0000 7f2e1621b640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f2e100cc190 0x7f2e100ca640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.108+0000 7f2e1621b640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2e10152250 0x7f2e100cab80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.108+0000 7f2e1621b640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f2e1015c400 con 0x7f2e10152250 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.108+0000 7f2e1621b640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f2e1015c280 con 0x7f2e100cc190 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.108+0000 7f2e1621b640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f2e1015c580 con 0x7f2e10073cd0 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.108+0000 7f2e0ffff640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2e10152250 0x7f2e100cab80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0ffff640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2e10152250 0x7f2e100cab80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:42024/0 (socket says 172.21.15.98:42024) 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0ffff640 1 -- 172.21.15.98:0/2329149590 learned_addr learned my addr 172.21.15.98:0/2329149590 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0effd640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f2e100cc190 0x7f2e100ca640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0f7fe640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f2e10073cd0 0x7f2e10150f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0ffff640 1 -- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f2e10073cd0 msgr2=0x7f2e10150f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0ffff640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f2e10073cd0 0x7f2e10150f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0ffff640 1 -- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f2e100cc190 msgr2=0x7f2e100ca640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:36.111 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0ffff640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f2e100cc190 0x7f2e100ca640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:36.112 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0ffff640 1 -- 172.21.15.98:0/2329149590 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2e10157700 con 0x7f2e10152250 2024-08-14T06:52:36.112 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0effd640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f2e100cc190 0x7f2e100ca640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-08-14T06:52:36.112 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e0ffff640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2e10152250 0x7f2e100cab80 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f2e0800cae0 tx=0x7f2e08007220 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:36.112 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2df7fff640 1 -- 172.21.15.98:0/2329149590 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e08007da0 con 0x7f2e10152250 2024-08-14T06:52:36.112 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2df7fff640 1 -- 172.21.15.98:0/2329149590 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2e08004c30 con 0x7f2e10152250 2024-08-14T06:52:36.112 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e1621b640 1 -- 172.21.15.98:0/2329149590 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2e101579e0 con 0x7f2e10152250 2024-08-14T06:52:36.112 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2df7fff640 1 -- 172.21.15.98:0/2329149590 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e08005280 con 0x7f2e10152250 2024-08-14T06:52:36.112 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.109+0000 7f2e1621b640 1 -- 172.21.15.98:0/2329149590 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2e10157fa0 con 0x7f2e10152250 2024-08-14T06:52:36.112 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.110+0000 7f2e1621b640 1 -- 172.21.15.98:0/2329149590 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2e10073790 con 0x7f2e10152250 2024-08-14T06:52:36.113 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.111+0000 7f2df7fff640 1 -- 172.21.15.98:0/2329149590 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 5) ==== 99139+0+0 (secure 0 0 0) 0x7f2e08005420 con 0x7f2e10152250 2024-08-14T06:52:36.113 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.111+0000 7f2df7fff640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f2dfc077040 0x7f2dfc079550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:36.113 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.111+0000 7f2df7fff640 1 -- 172.21.15.98:0/2329149590 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 4857+0+0 (secure 0 0 0) 0x7f2e08099070 con 0x7f2e10152250 2024-08-14T06:52:36.113 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.111+0000 7f2e0f7fe640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f2dfc077040 0x7f2dfc079550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:36.113 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.112+0000 7f2e0f7fe640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f2dfc077040 0x7f2dfc079550 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f2e00034000 tx=0x7f2e00004730 comp rx=0 tx=0).ready entity=mgr.4109 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:36.116 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.116+0000 7f2df7fff640 1 -- 172.21.15.98:0/2329149590 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+194988 (secure 0 0 0) 0x7f2e08014070 con 0x7f2e10152250 2024-08-14T06:52:36.304 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:36.303+0000 7f2e1621b640 1 -- 172.21.15.98:0/2329149590 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) -- 0x7f2e10158d10 con 0x7f2e10152250 2024-08-14T06:52:37.208 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.207+0000 7f2df7fff640 1 -- 172.21.15.98:0/2329149590 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v20) ==== 138+0+0 (secure 0 0 0) 0x7f2e08062540 con 0x7f2e10152250 2024-08-14T06:52:37.208 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'cephfs_metadata' created 2024-08-14T06:52:37.211 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.210+0000 7f2e1621b640 1 -- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f2dfc077040 msgr2=0x7f2dfc079550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:37.211 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.210+0000 7f2e1621b640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f2dfc077040 0x7f2dfc079550 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f2e00034000 tx=0x7f2e00004730 comp rx=0 tx=0).stop 2024-08-14T06:52:37.211 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.210+0000 7f2e1621b640 1 -- 172.21.15.98:0/2329149590 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2e10152250 msgr2=0x7f2e100cab80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:37.211 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.210+0000 7f2e1621b640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2e10152250 0x7f2e100cab80 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f2e0800cae0 tx=0x7f2e08007220 comp rx=0 tx=0).stop 2024-08-14T06:52:37.212 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.211+0000 7f2e1621b640 1 -- 172.21.15.98:0/2329149590 shutdown_connections 2024-08-14T06:52:37.212 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.211+0000 7f2e1621b640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f2dfc077040 0x7f2dfc079550 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:37.212 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.211+0000 7f2e1621b640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f2e10152250 0x7f2e100cab80 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:37.212 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.211+0000 7f2e1621b640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f2e100cc190 0x7f2e100ca640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:37.212 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.211+0000 7f2e1621b640 1 --2- 172.21.15.98:0/2329149590 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f2e10073cd0 0x7f2e10150f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:37.212 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.211+0000 7f2e1621b640 1 -- 172.21.15.98:0/2329149590 >> 172.21.15.98:0/2329149590 conn(0x7f2e100b2b30 msgr2=0x7f2e100afe20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:37.212 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.211+0000 7f2e1621b640 1 -- 172.21.15.98:0/2329149590 shutdown_connections 2024-08-14T06:52:37.212 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.211+0000 7f2e1621b640 1 -- 172.21.15.98:0/2329149590 wait complete. 2024-08-14T06:52:37.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-08-14T06:52:37.349 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.348+0000 7f40931b4640 1 -- 172.21.15.98:0/2943819502 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f408c0cc190 msgr2=0x7f408c073790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:37.349 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.348+0000 7f40931b4640 1 --2- 172.21.15.98:0/2943819502 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f408c0cc190 0x7f408c073790 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f408c073640 tx=0x7f407c02a910 comp rx=0 tx=0).stop 2024-08-14T06:52:37.349 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.348+0000 7f40931b4640 1 -- 172.21.15.98:0/2943819502 shutdown_connections 2024-08-14T06:52:37.349 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.348+0000 7f40931b4640 1 --2- 172.21.15.98:0/2943819502 >> v2:172.21.15.191:3300/0 conn(0x7f408c073cd0 0x7f408c151b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:37.350 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.348+0000 7f40931b4640 1 --2- 172.21.15.98:0/2943819502 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f408c152080 0x7f408c15bb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:37.350 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.348+0000 7f40931b4640 1 --2- 172.21.15.98:0/2943819502 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f408c0cc190 0x7f408c073790 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:37.350 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.348+0000 7f40931b4640 1 -- 172.21.15.98:0/2943819502 >> 172.21.15.98:0/2943819502 conn(0x7f408c0b2b30 msgr2=0x7f408c0b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:37.350 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.348+0000 7f40931b4640 1 -- 172.21.15.98:0/2943819502 shutdown_connections 2024-08-14T06:52:37.350 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.348+0000 7f40931b4640 1 -- 172.21.15.98:0/2943819502 wait complete. 2024-08-14T06:52:37.350 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.350+0000 7f40931b4640 1 Processor -- start 2024-08-14T06:52:37.350 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.350+0000 7f40931b4640 1 -- start start 2024-08-14T06:52:37.353 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.351+0000 7f40931b4640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f408c073cd0 0x7f408c1f2bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:37.353 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.351+0000 7f40931b4640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f408c0cc190 0x7f408c156880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:37.353 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.351+0000 7f40931b4640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f408c152080 0x7f408c150f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:37.353 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.351+0000 7f40931b4640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f408c15c1c0 con 0x7f408c152080 2024-08-14T06:52:37.353 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.351+0000 7f40931b4640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f408c15c040 con 0x7f408c0cc190 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.351+0000 7f40931b4640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f408c15c340 con 0x7f408c073cd0 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.351+0000 7f4091751640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f408c152080 0x7f408c150f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.351+0000 7f4091751640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f408c152080 0x7f408c150f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:42046/0 (socket says 172.21.15.98:42046) 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.351+0000 7f4091751640 1 -- 172.21.15.98:0/1434658847 learned_addr learned my addr 172.21.15.98:0/1434658847 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f4090f50640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f408c073cd0 0x7f408c1f2bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f408bfff640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f408c0cc190 0x7f408c156880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f4091751640 1 -- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f408c073cd0 msgr2=0x7f408c1f2bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f4091751640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f408c073cd0 0x7f408c1f2bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f4091751640 1 -- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f408c0cc190 msgr2=0x7f408c156880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f4091751640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f408c0cc190 0x7f408c156880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f4091751640 1 -- 172.21.15.98:0/1434658847 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f408c156dc0 con 0x7f408c152080 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f408bfff640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f408c0cc190 0x7f408c156880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-08-14T06:52:37.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f4091751640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f408c152080 0x7f408c150f40 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f4084007880 tx=0x7f4084007d40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:37.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f40897fa640 1 -- 172.21.15.98:0/1434658847 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f408400e630 con 0x7f408c152080 2024-08-14T06:52:37.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f40897fa640 1 -- 172.21.15.98:0/1434658847 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f408400e7f0 con 0x7f408c152080 2024-08-14T06:52:37.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f40931b4640 1 -- 172.21.15.98:0/1434658847 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f408c157450 con 0x7f408c152080 2024-08-14T06:52:37.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f40897fa640 1 -- 172.21.15.98:0/1434658847 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f408400ec40 con 0x7f408c152080 2024-08-14T06:52:37.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.352+0000 7f40931b4640 1 -- 172.21.15.98:0/1434658847 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f408c156080 con 0x7f408c152080 2024-08-14T06:52:37.356 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.353+0000 7f40931b4640 1 -- 172.21.15.98:0/1434658847 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f408c073790 con 0x7f408c152080 2024-08-14T06:52:37.356 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.354+0000 7f40897fa640 1 -- 172.21.15.98:0/1434658847 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 5) ==== 99139+0+0 (secure 0 0 0) 0x7f408400a440 con 0x7f408c152080 2024-08-14T06:52:37.356 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.354+0000 7f40897fa640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f406c077040 0x7f406c079550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:37.356 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.354+0000 7f40897fa640 1 -- 172.21.15.98:0/1434658847 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 5227+0+0 (secure 0 0 0) 0x7f408409a570 con 0x7f408c152080 2024-08-14T06:52:37.356 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.355+0000 7f4090f50640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f406c077040 0x7f406c079550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:37.356 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.355+0000 7f4090f50640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f406c077040 0x7f406c079550 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f407c035000 tx=0x7f407c004ab0 comp rx=0 tx=0).ready entity=mgr.4109 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:37.359 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.359+0000 7f40897fa640 1 -- 172.21.15.98:0/1434658847 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+194988 (secure 0 0 0) 0x7f40840638d0 con 0x7f408c152080 2024-08-14T06:52:37.543 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:37.543+0000 7f40931b4640 1 -- 172.21.15.98:0/1434658847 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) -- 0x7f408c074aa0 con 0x7f408c152080 2024-08-14T06:52:38.210 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.209+0000 7f40897fa640 1 -- 172.21.15.98:0/1434658847 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v21) ==== 170+0+0 (secure 0 0 0) 0x7f40840631d0 con 0x7f408c152080 2024-08-14T06:52:38.210 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'cephfs_data' created 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 -- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f406c077040 msgr2=0x7f406c079550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f406c077040 0x7f406c079550 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f407c035000 tx=0x7f407c004ab0 comp rx=0 tx=0).stop 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 -- 172.21.15.98:0/1434658847 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f408c152080 msgr2=0x7f408c150f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f408c152080 0x7f408c150f40 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f4084007880 tx=0x7f4084007d40 comp rx=0 tx=0).stop 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 -- 172.21.15.98:0/1434658847 shutdown_connections 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f406c077040 0x7f406c079550 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f408c152080 0x7f408c150f40 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f408c0cc190 0x7f408c156880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 --2- 172.21.15.98:0/1434658847 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f408c073cd0 0x7f408c1f2bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 -- 172.21.15.98:0/1434658847 >> 172.21.15.98:0/1434658847 conn(0x7f408c0b2b30 msgr2=0x7f408c0cf2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:38.213 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 -- 172.21.15.98:0/1434658847 shutdown_connections 2024-08-14T06:52:38.214 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.212+0000 7f40931b4640 1 -- 172.21.15.98:0/1434658847 wait complete. 2024-08-14T06:52:38.233 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-08-14T06:52:38.353 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.352+0000 7f84b35a8640 1 -- 172.21.15.98:0/3019585641 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f84ac160c30 msgr2=0x7f84ac15c440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:38.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.352+0000 7f84b35a8640 1 --2- 172.21.15.98:0/3019585641 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f84ac160c30 0x7f84ac15c440 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f84a8005fd0 tx=0x7f84a802a670 comp rx=0 tx=0).stop 2024-08-14T06:52:38.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.352+0000 7f84b35a8640 1 -- 172.21.15.98:0/3019585641 shutdown_connections 2024-08-14T06:52:38.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.352+0000 7f84b35a8640 1 --2- 172.21.15.98:0/3019585641 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f84ac160c30 0x7f84ac15c440 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:38.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.352+0000 7f84b35a8640 1 -- 172.21.15.98:0/3019585641 >> 172.21.15.98:0/3019585641 conn(0x7f84ac0b2b30 msgr2=0x7f84ac0b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:38.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.352+0000 7f84b35a8640 1 -- 172.21.15.98:0/3019585641 shutdown_connections 2024-08-14T06:52:38.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.352+0000 7f84b35a8640 1 -- 172.21.15.98:0/3019585641 wait complete. 2024-08-14T06:52:38.354 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.354+0000 7f84b35a8640 1 Processor -- start 2024-08-14T06:52:38.355 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.354+0000 7f84b35a8640 1 -- start start 2024-08-14T06:52:38.357 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.355+0000 7f84b35a8640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f84ac0d0530 0x7f84ac0b1140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:38.357 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.355+0000 7f84b35a8640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f84ac1f3580 0x7f84ac1f3990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:38.357 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.355+0000 7f84b35a8640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f84ac0aef90 0x7f84ac0af3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.355+0000 7f84b35a8640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f84ac074230 con 0x7f84ac0aef90 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.355+0000 7f84b35a8640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f84ac0740b0 con 0x7f84ac1f3580 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.355+0000 7f84b35a8640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f84ac0743b0 con 0x7f84ac0d0530 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b1b45640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f84ac0aef90 0x7f84ac0af3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b1b45640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f84ac0aef90 0x7f84ac0af3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:42064/0 (socket says 172.21.15.98:42064) 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b1b45640 1 -- 172.21.15.98:0/1255905191 learned_addr learned my addr 172.21.15.98:0/1255905191 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b0b43640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f84ac1f3580 0x7f84ac1f3990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b1344640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f84ac0d0530 0x7f84ac0b1140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b1b45640 1 -- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f84ac0d0530 msgr2=0x7f84ac0b1140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b1b45640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f84ac0d0530 0x7f84ac0b1140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b1b45640 1 -- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f84ac1f3580 msgr2=0x7f84ac1f3990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b1b45640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f84ac1f3580 0x7f84ac1f3990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:38.358 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b1b45640 1 -- 172.21.15.98:0/1255905191 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f84ac0af8e0 con 0x7f84ac0aef90 2024-08-14T06:52:38.359 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b1b45640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f84ac0aef90 0x7f84ac0af3a0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f84a400c760 tx=0x7f84a400cb60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:38.359 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84a1ffb640 1 -- 172.21.15.98:0/1255905191 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84a4003b70 con 0x7f84ac0aef90 2024-08-14T06:52:38.359 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84a1ffb640 1 -- 172.21.15.98:0/1255905191 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f84a4003d10 con 0x7f84ac0aef90 2024-08-14T06:52:38.359 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b35a8640 1 -- 172.21.15.98:0/1255905191 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f84ac153070 con 0x7f84ac0aef90 2024-08-14T06:52:38.359 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84a1ffb640 1 -- 172.21.15.98:0/1255905191 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84a40052d0 con 0x7f84ac0aef90 2024-08-14T06:52:38.359 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.356+0000 7f84b35a8640 1 -- 172.21.15.98:0/1255905191 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f84ac153630 con 0x7f84ac0aef90 2024-08-14T06:52:38.360 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.357+0000 7f84b35a8640 1 -- 172.21.15.98:0/1255905191 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f84ac15bfb0 con 0x7f84ac0aef90 2024-08-14T06:52:38.360 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.358+0000 7f84a1ffb640 1 -- 172.21.15.98:0/1255905191 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 5) ==== 99139+0+0 (secure 0 0 0) 0x7f84a4005470 con 0x7f84ac0aef90 2024-08-14T06:52:38.360 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.358+0000 7f84a1ffb640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f8490077010 0x7f8490079520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:38.360 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.358+0000 7f84a1ffb640 1 -- 172.21.15.98:0/1255905191 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 5609+0+0 (secure 0 0 0) 0x7f84a40986d0 con 0x7f84ac0aef90 2024-08-14T06:52:38.360 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.359+0000 7f84b1344640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f8490077010 0x7f8490079520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:38.360 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.359+0000 7f84b1344640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f8490077010 0x7f8490079520 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f84a8033000 tx=0x7f84a8004730 comp rx=0 tx=0).ready entity=mgr.4109 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:38.363 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.363+0000 7f84a1ffb640 1 -- 172.21.15.98:0/1255905191 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+194988 (secure 0 0 0) 0x7f84ac0af8e0 con 0x7f84ac0aef90 2024-08-14T06:52:38.587 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:38.587+0000 7f84b35a8640 1 -- 172.21.15.98:0/1255905191 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) -- 0x7f84ac15bcd0 con 0x7f84ac0aef90 2024-08-14T06:52:39.236 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.235+0000 7f84a1ffb640 1 -- 172.21.15.98:0/1255905191 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 2 and data pool 3 v4) ==== 172+0+0 (secure 0 0 0) 0x7f84a4061930 con 0x7f84ac0aef90 2024-08-14T06:52:39.237 INFO:teuthology.orchestra.run.smithi098.stderr:new fs with metadata pool 2 and data pool 3 2024-08-14T06:52:39.249 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.248+0000 7f84b35a8640 1 -- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f8490077010 msgr2=0x7f8490079520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.249 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.248+0000 7f84b35a8640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f8490077010 0x7f8490079520 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f84a8033000 tx=0x7f84a8004730 comp rx=0 tx=0).stop 2024-08-14T06:52:39.249 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.248+0000 7f84b35a8640 1 -- 172.21.15.98:0/1255905191 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f84ac0aef90 msgr2=0x7f84ac0af3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.249 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.248+0000 7f84b35a8640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f84ac0aef90 0x7f84ac0af3a0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f84a400c760 tx=0x7f84a400cb60 comp rx=0 tx=0).stop 2024-08-14T06:52:39.250 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.248+0000 7f84b35a8640 1 -- 172.21.15.98:0/1255905191 shutdown_connections 2024-08-14T06:52:39.250 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.248+0000 7f84b35a8640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f8490077010 0x7f8490079520 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.250 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.248+0000 7f84b35a8640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f84ac0aef90 0x7f84ac0af3a0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.250 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.248+0000 7f84b35a8640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f84ac1f3580 0x7f84ac1f3990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.250 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.248+0000 7f84b35a8640 1 --2- 172.21.15.98:0/1255905191 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f84ac0d0530 0x7f84ac0b1140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.250 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.248+0000 7f84b35a8640 1 -- 172.21.15.98:0/1255905191 >> 172.21.15.98:0/1255905191 conn(0x7f84ac0b2b30 msgr2=0x7f84ac154820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:39.251 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.249+0000 7f84b35a8640 1 -- 172.21.15.98:0/1255905191 shutdown_connections 2024-08-14T06:52:39.251 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.249+0000 7f84b35a8640 1 -- 172.21.15.98:0/1255905191 wait complete. 2024-08-14T06:52:39.267 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-08-14T06:52:39.383 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.381+0000 7fb939384640 1 -- 172.21.15.98:0/2162098120 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb920004800 msgr2=0x7fb920004c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.383 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.381+0000 7fb939384640 1 --2- 172.21.15.98:0/2162098120 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb920004800 0x7fb920004c10 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fb91c006fd0 tx=0x7fb920005830 comp rx=0 tx=0).stop 2024-08-14T06:52:39.383 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.381+0000 7fb939384640 1 -- 172.21.15.98:0/2162098120 shutdown_connections 2024-08-14T06:52:39.383 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.381+0000 7fb939384640 1 --2- 172.21.15.98:0/2162098120 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb920004800 0x7fb920004c10 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.383 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.381+0000 7fb939384640 1 -- 172.21.15.98:0/2162098120 >> 172.21.15.98:0/2162098120 conn(0x7fb93409d540 msgr2=0x7fb9340d42e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:39.384 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.381+0000 7fb939384640 1 -- 172.21.15.98:0/2162098120 shutdown_connections 2024-08-14T06:52:39.384 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.381+0000 7fb939384640 1 -- 172.21.15.98:0/2162098120 wait complete. 2024-08-14T06:52:39.388 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.383+0000 7fb939384640 1 Processor -- start 2024-08-14T06:52:39.388 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.383+0000 7fb939384640 1 -- start start 2024-08-14T06:52:39.388 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.384+0000 7fb939384640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb920004800 0x7fb9341f7f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:39.389 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.384+0000 7fb939384640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb9341e9300 0x7fb9341e9710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:39.389 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.384+0000 7fb939384640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb9341f8720 0x7fb9341f8b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:39.389 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.384+0000 7fb939384640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7fb9340d2650 con 0x7fb920004800 2024-08-14T06:52:39.389 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.384+0000 7fb939384640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7fb9340d24d0 con 0x7fb9341f8720 2024-08-14T06:52:39.389 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.384+0000 7fb939384640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7fb9340d27d0 con 0x7fb9341e9300 2024-08-14T06:52:39.389 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.384+0000 7fb932ffd640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb920004800 0x7fb9341f7f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:39.389 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.384+0000 7fb932ffd640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb920004800 0x7fb9341f7f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:42092/0 (socket says 172.21.15.98:42092) 2024-08-14T06:52:39.389 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.384+0000 7fb932ffd640 1 -- 172.21.15.98:0/2294940561 learned_addr learned my addr 172.21.15.98:0/2294940561 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.384+0000 7fb9337fe640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb9341f8720 0x7fb9341f8b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb92a7fc640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb9341e9300 0x7fb9341e9710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb932ffd640 1 -- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb9341e9300 msgr2=0x7fb9341e9710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb932ffd640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb9341e9300 0x7fb9341e9710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb932ffd640 1 -- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb9341f8720 msgr2=0x7fb9341f8b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb932ffd640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb9341f8720 0x7fb9341f8b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb932ffd640 1 -- 172.21.15.98:0/2294940561 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb9341f9070 con 0x7fb920004800 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb9337fe640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb9341f8720 0x7fb9341f8b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb932ffd640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb920004800 0x7fb9341f7f20 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fb924003f00 tx=0x7fb92400f720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb92bfff640 1 -- 172.21.15.98:0/2294940561 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb924015070 con 0x7fb920004800 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb92bfff640 1 -- 172.21.15.98:0/2294940561 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb924011840 con 0x7fb920004800 2024-08-14T06:52:39.390 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb939384640 1 -- 172.21.15.98:0/2294940561 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb9341ea050 con 0x7fb920004800 2024-08-14T06:52:39.391 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb92bfff640 1 -- 172.21.15.98:0/2294940561 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb924011c70 con 0x7fb920004800 2024-08-14T06:52:39.391 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.385+0000 7fb939384640 1 -- 172.21.15.98:0/2294940561 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb9341ea610 con 0x7fb920004800 2024-08-14T06:52:39.391 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.386+0000 7fb939384640 1 -- 172.21.15.98:0/2294940561 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb9340cfc30 con 0x7fb920004800 2024-08-14T06:52:39.391 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.387+0000 7fb92bfff640 1 -- 172.21.15.98:0/2294940561 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 5) ==== 99139+0+0 (secure 0 0 0) 0x7fb9240103c0 con 0x7fb920004800 2024-08-14T06:52:39.391 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.387+0000 7fb92bfff640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fb914077010 0x7fb914079520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:39.391 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.387+0000 7fb92bfff640 1 -- 172.21.15.98:0/2294940561 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(22..22 src has 1..22) ==== 5709+0+0 (secure 0 0 0) 0x7fb924099360 con 0x7fb920004800 2024-08-14T06:52:39.391 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.387+0000 7fb92a7fc640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fb914077010 0x7fb914079520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:39.391 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.388+0000 7fb92a7fc640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fb914077010 0x7fb914079520 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fb91c033000 tx=0x7fb9340b39f0 comp rx=0 tx=0).ready entity=mgr.4109 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:39.392 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.392+0000 7fb92bfff640 1 -- 172.21.15.98:0/2294940561 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+194988 (secure 0 0 0) 0x7fb9341f9070 con 0x7fb920004800 2024-08-14T06:52:39.578 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.578+0000 7fb939384640 1 -- 172.21.15.98:0/2294940561 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fb93408e2f0 con 0x7fb920004800 2024-08-14T06:52:39.579 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.579+0000 7fb92bfff640 1 -- 172.21.15.98:0/2294940561 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) ==== 74+0+16706 (secure 0 0 0) 0x7fb9240625f0 con 0x7fb920004800 2024-08-14T06:52:39.580 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:52:39.580 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":22,"fsid":"1a641e76-d282-43fb-b02c-28f3852c1ab1","created":"2024-08-14T06:52:14.033527+0000","modified":"2024-08-14T06:52:38.588355+0000","last_up_change":"2024-08-14T06:52:27.155904+0000","last_in_change":"2024-08-14T06:52:17.300583+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-14T06:52:28.459302+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-08-14T06:52:36.307636+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-08-14T06:52:37.547190+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"273f37e4-1d4c-46cd-96bc-3d7fce969572","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6809","nonce":3122853447}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6811","nonce":3122853447}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6815","nonce":3122853447}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6813","nonce":3122853447}]},"public_addr":"172.21.15.98:6809/3122853447","cluster_addr":"172.21.15.98:6811/3122853447","heartbeat_back_addr":"172.21.15.98:6815/3122853447","heartbeat_front_addr":"172.21.15.98:6813/3122853447","state":["exists","up"]},{"osd":1,"uuid":"ebd9bc08-f436-4e26-a0ba-d79a2220ba5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6817","nonce":4048855513}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6819","nonce":4048855513}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6823","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6825","nonce":4048855513}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6821","nonce":4048855513}]},"public_addr":"172.21.15.98:6817/4048855513","cluster_addr":"172.21.15.98:6819/4048855513","heartbeat_back_addr":"172.21.15.98:6825/4048855513","heartbeat_front_addr":"172.21.15.98:6821/4048855513","state":["exists","up"]},{"osd":2,"uuid":"9979a52a-0573-4019-a471-2ad384d88176","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6824","nonce":2997064267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6827","nonce":2997064267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6831","nonce":2997064267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6829","nonce":2997064267}]},"public_addr":"172.21.15.98:6824/2997064267","cluster_addr":"172.21.15.98:6827/2997064267","heartbeat_back_addr":"172.21.15.98:6831/2997064267","heartbeat_front_addr":"172.21.15.98:6829/2997064267","state":["exists","up"]},{"osd":3,"uuid":"aa26d769-fd1e-4963-af87-a7006f935553","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6800","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6801","nonce":3163598583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6803","nonce":3163598583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6807","nonce":3163598583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6805","nonce":3163598583}]},"public_addr":"172.21.15.98:6801/3163598583","cluster_addr":"172.21.15.98:6803/3163598583","heartbeat_back_addr":"172.21.15.98:6807/3163598583","heartbeat_front_addr":"172.21.15.98:6805/3163598583","state":["exists","up"]},{"osd":4,"uuid":"0cd7a5fd-20b0-40f2-9e45-fd71ceadb6e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6819","nonce":3843105526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6821","nonce":3843105526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6825","nonce":3843105526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6823","nonce":3843105526}]},"public_addr":"172.21.15.191:6819/3843105526","cluster_addr":"172.21.15.191:6821/3843105526","heartbeat_back_addr":"172.21.15.191:6825/3843105526","heartbeat_front_addr":"172.21.15.191:6823/3843105526","state":["exists","up"]},{"osd":5,"uuid":"fbc7b048-ee2c-46e1-81e9-854d0f132434","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6827","nonce":2246847695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6829","nonce":2246847695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6832","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6833","nonce":2246847695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6831","nonce":2246847695}]},"public_addr":"172.21.15.191:6827/2246847695","cluster_addr":"172.21.15.191:6829/2246847695","heartbeat_back_addr":"172.21.15.191:6833/2246847695","heartbeat_front_addr":"172.21.15.191:6831/2246847695","state":["exists","up"]},{"osd":6,"uuid":"3f5d0d02-eae3-4d2b-b611-03cbec6590a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6811","nonce":2491745875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6813","nonce":2491745875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6817","nonce":2491745875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6815","nonce":2491745875}]},"public_addr":"172.21.15.191:6811/2491745875","cluster_addr":"172.21.15.191:6813/2491745875","heartbeat_back_addr":"172.21.15.191:6817/2491745875","heartbeat_front_addr":"172.21.15.191:6815/2491745875","state":["exists","up"]},{"osd":7,"uuid":"8d15135a-d5ad-47b1-988a-c01ccd36426e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6801","nonce":2707254681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6803","nonce":2707254681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6807","nonce":2707254681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6805","nonce":2707254681}]},"public_addr":"172.21.15.191:6801/2707254681","cluster_addr":"172.21.15.191:6803/2707254681","heartbeat_back_addr":"172.21.15.191:6807/2707254681","heartbeat_front_addr":"172.21.15.191:6805/2707254681","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.934566+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.154949+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.113670+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.445209+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.574716+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.080765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.505776+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.103787+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-14T06:52:39.583 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 -- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fb914077010 msgr2=0x7fb914079520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.583 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fb914077010 0x7fb914079520 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fb91c033000 tx=0x7fb9340b39f0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.583 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 -- 172.21.15.98:0/2294940561 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb920004800 msgr2=0x7fb9341f7f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.583 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb920004800 0x7fb9341f7f20 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fb924003f00 tx=0x7fb92400f720 comp rx=0 tx=0).stop 2024-08-14T06:52:39.584 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 -- 172.21.15.98:0/2294940561 shutdown_connections 2024-08-14T06:52:39.584 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7fb914077010 0x7fb914079520 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.584 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7fb9341f8720 0x7fb9341f8b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.584 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7fb9341e9300 0x7fb9341e9710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.584 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 --2- 172.21.15.98:0/2294940561 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7fb920004800 0x7fb9341f7f20 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.584 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 -- 172.21.15.98:0/2294940561 >> 172.21.15.98:0/2294940561 conn(0x7fb93409d540 msgr2=0x7fb93406d770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:39.585 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 -- 172.21.15.98:0/2294940561 shutdown_connections 2024-08-14T06:52:39.585 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.581+0000 7fb939384640 1 -- 172.21.15.98:0/2294940561 wait complete. 2024-08-14T06:52:39.593 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-08-14T06:52:39.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.706+0000 7f5cafc6d640 1 -- 172.21.15.98:0/1534784741 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5ca80d18c0 msgr2=0x7f5ca80d4110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.706+0000 7f5cafc6d640 1 --2- 172.21.15.98:0/1534784741 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5ca80d18c0 0x7f5ca80d4110 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f5ca4009830 tx=0x7f5ca402a510 comp rx=0 tx=0).stop 2024-08-14T06:52:39.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.706+0000 7f5cafc6d640 1 -- 172.21.15.98:0/1534784741 shutdown_connections 2024-08-14T06:52:39.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.706+0000 7f5cafc6d640 1 --2- 172.21.15.98:0/1534784741 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f5ca80d4650 0x7f5ca80b7c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.706+0000 7f5cafc6d640 1 --2- 172.21.15.98:0/1534784741 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5ca80d18c0 0x7f5ca80d4110 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.706+0000 7f5cafc6d640 1 -- 172.21.15.98:0/1534784741 >> 172.21.15.98:0/1534784741 conn(0x7f5ca808d9d0 msgr2=0x7f5ca809d600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-08-14T06:52:39.707 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.706+0000 7f5cafc6d640 1 -- 172.21.15.98:0/1534784741 shutdown_connections 2024-08-14T06:52:39.708 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.706+0000 7f5cafc6d640 1 -- 172.21.15.98:0/1534784741 wait complete. 2024-08-14T06:52:39.708 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.708+0000 7f5cafc6d640 1 Processor -- start 2024-08-14T06:52:39.708 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.708+0000 7f5cafc6d640 1 -- start start 2024-08-14T06:52:39.711 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cafc6d640 1 --2- >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5ca80b81d0 0x7f5ca80a1b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:39.711 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cafc6d640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5ca80d18c0 0x7f5ca809ec70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:39.711 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cafc6d640 1 --2- >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f5ca80d4650 0x7f5ca80b21a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:39.712 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cafc6d640 1 -- --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_getmap magic: 0 -- 0x7f5ca80b54c0 con 0x7f5ca80d18c0 2024-08-14T06:52:39.712 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cafc6d640 1 -- --> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] -- mon_getmap magic: 0 -- 0x7f5ca80b8f00 con 0x7f5ca80b81d0 2024-08-14T06:52:39.712 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cafc6d640 1 -- --> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] -- mon_getmap magic: 0 -- 0x7f5ca80b5640 con 0x7f5ca80d4650 2024-08-14T06:52:39.712 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cad208640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5ca80d18c0 0x7f5ca809ec70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:39.712 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cad208640 1 --2- >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5ca80d18c0 0x7f5ca809ec70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.98:3300/0 says I am v2:172.21.15.98:42096/0 (socket says 172.21.15.98:42096) 2024-08-14T06:52:39.712 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cad208640 1 -- 172.21.15.98:0/2085336853 learned_addr learned my addr 172.21.15.98:0/2085336853 (peer_addr_for_me v2:172.21.15.98:0/0) 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cada09640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5ca80b81d0 0x7f5ca80a1b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cae20a640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f5ca80d4650 0x7f5ca80b21a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cad208640 1 -- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f5ca80d4650 msgr2=0x7f5ca80b21a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cad208640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f5ca80d4650 0x7f5ca80b21a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cad208640 1 -- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5ca80b81d0 msgr2=0x7f5ca80a1b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cad208640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x7f5ca80b81d0 0x7f5ca80a1b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cad208640 1 -- 172.21.15.98:0/2085336853 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5ca809f650 con 0x7f5ca80d18c0 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cae20a640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:3301/0,v1:172.21.15.191:6790/0] conn(0x7f5ca80d4650 0x7f5ca80b21a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.709+0000 7f5cad208640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5ca80d18c0 0x7f5ca809ec70 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f5c98009fd0 tx=0x7f5c9800ec20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.710+0000 7f5c9e7fc640 1 -- 172.21.15.98:0/2085336853 <== mon.0 v2:172.21.15.98:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5c9800ccc0 con 0x7f5ca80d18c0 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.710+0000 7f5c9e7fc640 1 -- 172.21.15.98:0/2085336853 <== mon.0 v2:172.21.15.98:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5c9800ee70 con 0x7f5ca80d18c0 2024-08-14T06:52:39.713 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.710+0000 7f5cafc6d640 1 -- 172.21.15.98:0/2085336853 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5ca80a1030 con 0x7f5ca80d18c0 2024-08-14T06:52:39.714 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.710+0000 7f5c9e7fc640 1 -- 172.21.15.98:0/2085336853 <== mon.0 v2:172.21.15.98:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5c9800ce60 con 0x7f5ca80d18c0 2024-08-14T06:52:39.714 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.710+0000 7f5cafc6d640 1 -- 172.21.15.98:0/2085336853 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5ca80a15f0 con 0x7f5ca80d18c0 2024-08-14T06:52:39.714 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.711+0000 7f5cafc6d640 1 -- 172.21.15.98:0/2085336853 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5ca80d4110 con 0x7f5ca80d18c0 2024-08-14T06:52:39.715 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.711+0000 7f5c9e7fc640 1 -- 172.21.15.98:0/2085336853 <== mon.0 v2:172.21.15.98:3300/0 4 ==== mgrmap(e 5) ==== 99139+0+0 (secure 0 0 0) 0x7f5c98005260 con 0x7f5ca80d18c0 2024-08-14T06:52:39.716 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.712+0000 7f5c9e7fc640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f5c8c077010 0x7f5c8c079520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-08-14T06:52:39.716 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.712+0000 7f5c9e7fc640 1 -- 172.21.15.98:0/2085336853 <== mon.0 v2:172.21.15.98:3300/0 5 ==== osd_map(22..22 src has 1..22) ==== 5709+0+0 (secure 0 0 0) 0x7f5c98099130 con 0x7f5ca80d18c0 2024-08-14T06:52:39.716 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.712+0000 7f5cada09640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f5c8c077010 0x7f5c8c079520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-08-14T06:52:39.716 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.713+0000 7f5cada09640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f5c8c077010 0x7f5c8c079520 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f5ca4034000 tx=0x7f5ca4004730 comp rx=0 tx=0).ready entity=mgr.4109 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-08-14T06:52:39.717 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.716+0000 7f5c9e7fc640 1 -- 172.21.15.98:0/2085336853 <== mon.0 v2:172.21.15.98:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+194988 (secure 0 0 0) 0x7f5c98062340 con 0x7f5ca80d18c0 2024-08-14T06:52:39.905 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.904+0000 7f5cafc6d640 1 -- 172.21.15.98:0/2085336853 --> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f5ca809f3e0 con 0x7f5ca80d18c0 2024-08-14T06:52:39.906 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.905+0000 7f5c9e7fc640 1 -- 172.21.15.98:0/2085336853 <== mon.0 v2:172.21.15.98:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) ==== 74+0+16706 (secure 0 0 0) 0x7f5c98061c40 con 0x7f5ca80d18c0 2024-08-14T06:52:39.906 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-08-14T06:52:39.907 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":22,"fsid":"1a641e76-d282-43fb-b02c-28f3852c1ab1","created":"2024-08-14T06:52:14.033527+0000","modified":"2024-08-14T06:52:38.588355+0000","last_up_change":"2024-08-14T06:52:27.155904+0000","last_in_change":"2024-08-14T06:52:17.300583+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-14T06:52:28.459302+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-08-14T06:52:36.307636+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-08-14T06:52:37.547190+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"273f37e4-1d4c-46cd-96bc-3d7fce969572","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6809","nonce":3122853447}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6811","nonce":3122853447}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6815","nonce":3122853447}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":3122853447},{"type":"v1","addr":"172.21.15.98:6813","nonce":3122853447}]},"public_addr":"172.21.15.98:6809/3122853447","cluster_addr":"172.21.15.98:6811/3122853447","heartbeat_back_addr":"172.21.15.98:6815/3122853447","heartbeat_front_addr":"172.21.15.98:6813/3122853447","state":["exists","up"]},{"osd":1,"uuid":"ebd9bc08-f436-4e26-a0ba-d79a2220ba5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6817","nonce":4048855513}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6819","nonce":4048855513}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6823","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6825","nonce":4048855513}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":4048855513},{"type":"v1","addr":"172.21.15.98:6821","nonce":4048855513}]},"public_addr":"172.21.15.98:6817/4048855513","cluster_addr":"172.21.15.98:6819/4048855513","heartbeat_back_addr":"172.21.15.98:6825/4048855513","heartbeat_front_addr":"172.21.15.98:6821/4048855513","state":["exists","up"]},{"osd":2,"uuid":"9979a52a-0573-4019-a471-2ad384d88176","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6824","nonce":2997064267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6827","nonce":2997064267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6831","nonce":2997064267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":2997064267},{"type":"v1","addr":"172.21.15.98:6829","nonce":2997064267}]},"public_addr":"172.21.15.98:6824/2997064267","cluster_addr":"172.21.15.98:6827/2997064267","heartbeat_back_addr":"172.21.15.98:6831/2997064267","heartbeat_front_addr":"172.21.15.98:6829/2997064267","state":["exists","up"]},{"osd":3,"uuid":"aa26d769-fd1e-4963-af87-a7006f935553","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6800","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6801","nonce":3163598583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6803","nonce":3163598583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6807","nonce":3163598583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":3163598583},{"type":"v1","addr":"172.21.15.98:6805","nonce":3163598583}]},"public_addr":"172.21.15.98:6801/3163598583","cluster_addr":"172.21.15.98:6803/3163598583","heartbeat_back_addr":"172.21.15.98:6807/3163598583","heartbeat_front_addr":"172.21.15.98:6805/3163598583","state":["exists","up"]},{"osd":4,"uuid":"0cd7a5fd-20b0-40f2-9e45-fd71ceadb6e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6819","nonce":3843105526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6821","nonce":3843105526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6825","nonce":3843105526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":3843105526},{"type":"v1","addr":"172.21.15.191:6823","nonce":3843105526}]},"public_addr":"172.21.15.191:6819/3843105526","cluster_addr":"172.21.15.191:6821/3843105526","heartbeat_back_addr":"172.21.15.191:6825/3843105526","heartbeat_front_addr":"172.21.15.191:6823/3843105526","state":["exists","up"]},{"osd":5,"uuid":"fbc7b048-ee2c-46e1-81e9-854d0f132434","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6827","nonce":2246847695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6829","nonce":2246847695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6832","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6833","nonce":2246847695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":2246847695},{"type":"v1","addr":"172.21.15.191:6831","nonce":2246847695}]},"public_addr":"172.21.15.191:6827/2246847695","cluster_addr":"172.21.15.191:6829/2246847695","heartbeat_back_addr":"172.21.15.191:6833/2246847695","heartbeat_front_addr":"172.21.15.191:6831/2246847695","state":["exists","up"]},{"osd":6,"uuid":"3f5d0d02-eae3-4d2b-b611-03cbec6590a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6811","nonce":2491745875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6813","nonce":2491745875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6817","nonce":2491745875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":2491745875},{"type":"v1","addr":"172.21.15.191:6815","nonce":2491745875}]},"public_addr":"172.21.15.191:6811/2491745875","cluster_addr":"172.21.15.191:6813/2491745875","heartbeat_back_addr":"172.21.15.191:6817/2491745875","heartbeat_front_addr":"172.21.15.191:6815/2491745875","state":["exists","up"]},{"osd":7,"uuid":"8d15135a-d5ad-47b1-988a-c01ccd36426e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6801","nonce":2707254681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6803","nonce":2707254681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6807","nonce":2707254681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2707254681},{"type":"v1","addr":"172.21.15.191:6805","nonce":2707254681}]},"public_addr":"172.21.15.191:6801/2707254681","cluster_addr":"172.21.15.191:6803/2707254681","heartbeat_back_addr":"172.21.15.191:6807/2707254681","heartbeat_front_addr":"172.21.15.191:6805/2707254681","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.934566+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.154949+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.113670+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.445209+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.574716+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:24.080765+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.505776+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-14T06:52:23.103787+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-14T06:52:39.909 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.908+0000 7f5cafc6d640 1 -- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f5c8c077010 msgr2=0x7f5c8c079520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.909 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.908+0000 7f5cafc6d640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.191:6808/680949956,v1:172.21.15.191:6809/680949956] conn(0x7f5c8c077010 0x7f5c8c079520 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f5ca4034000 tx=0x7f5ca4004730 comp rx=0 tx=0).stop 2024-08-14T06:52:39.909 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.908+0000 7f5cafc6d640 1 -- 172.21.15.98:0/2085336853 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5ca80d18c0 msgr2=0x7f5ca809ec70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-08-14T06:52:39.909 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.908+0000 7f5cafc6d640 1 --2- 172.21.15.98:0/2085336853 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x7f5ca80d18c0 0x7f5ca809ec70 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f5c98009fd0 tx=0x7f5c9800ec20 comp rx=0 tx=0).stop 2024-08-14T06:52:39.909 INFO:teuthology.orchestra.run.smithi098.stderr:2024-08-14T06:52:39.908+0000 7f5cafc6d640 1 -- 172.21.15.98:0/2085336853